From da86966855eccfee87cb16fc15338c96e1b4034e Mon Sep 17 00:00:00 2001 From: Andriy Kornatskyy Date: Tue, 30 Jun 2020 20:35:04 +0300 Subject: [PATCH] Improved linting with autoflake, black and isort. --- setup.py | 94 +++-- src/wheezy/__init__.py | 3 +- src/wheezy/caching/__init__.py | 8 +- src/wheezy/caching/client.py | 31 +- src/wheezy/caching/comp.py | 7 +- src/wheezy/caching/dependency.py | 35 +- src/wheezy/caching/encoding.py | 11 +- src/wheezy/caching/lockout.py | 68 ++-- src/wheezy/caching/logging.py | 4 +- src/wheezy/caching/memcache.py | 17 +- src/wheezy/caching/memory.py | 17 +- src/wheezy/caching/null.py | 1 - src/wheezy/caching/patterns.py | 125 ++++--- src/wheezy/caching/pylibmc.py | 25 +- src/wheezy/caching/tests/test_cache.py | 107 +++--- src/wheezy/caching/tests/test_client.py | 16 +- src/wheezy/caching/tests/test_dependency.py | 131 ++++--- src/wheezy/caching/tests/test_lockout.py | 245 ++++++------ src/wheezy/caching/tests/test_logging.py | 7 +- src/wheezy/caching/tests/test_memcache.py | 15 +- src/wheezy/caching/tests/test_memory.py | 2 - src/wheezy/caching/tests/test_patterns.py | 393 +++++++++++--------- src/wheezy/caching/tests/test_pylibmc.py | 17 +- src/wheezy/caching/utils.py | 6 +- tox.ini | 7 + 25 files changed, 740 insertions(+), 652 deletions(-) diff --git a/setup.py b/setup.py index 7805059..fad7ea0 100644 --- a/setup.py +++ b/setup.py @@ -8,72 +8,70 @@ extra = {} try: from Cython.Build import cythonize - p = os.path.join('src', 'wheezy', 'caching') - extra['ext_modules'] = cythonize( - [os.path.join(p, '*.py')], - exclude=os.path.join(p, '__init__.py'), - nthreads=2, quiet=True) + + p = os.path.join("src", "wheezy", "caching") + extra["ext_modules"] = cythonize( + [os.path.join(p, "*.py")], + exclude=os.path.join(p, "__init__.py"), + nthreads=2, + quiet=True, + ) except ImportError: pass -README = open(os.path.join(os.path.dirname(__file__), 'README.md')).read() +README = open(os.path.join(os.path.dirname(__file__), "README.md")).read() VERSION = ( re.search( - r"__version__ = '(.+)'", - open("src/wheezy/caching/__init__.py").read(), + r'__version__ = "(.+)"', open("src/wheezy/caching/__init__.py").read(), ) .group(1) .strip() ) setup( - name='wheezy.caching', + name="wheezy.caching", version=VERSION, - description='A lightweight caching library', + description="A lightweight caching library", long_description=README, - long_description_content_type='text/markdown', - url='https://github.com/akornatskyy/wheezy.caching', - author='Andriy Kornatskyy', - author_email='andriy.kornatskyy@live.com', - license='MIT', + long_description_content_type="text/markdown", + url="https://github.com/akornatskyy/wheezy.caching", + author="Andriy Kornatskyy", + author_email="andriy.kornatskyy@live.com", + license="MIT", classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.4', - 'Programming Language :: Python :: 2.5', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Software Development :: Libraries :: Python Modules' + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.4", + "Programming Language :: Python :: 2.5", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Software Development :: Libraries :: Python Modules", ], - keywords='caching dependency memory null memcache memcached pylibmc', - packages=['wheezy', 'wheezy.caching'], - package_dir={'': 'src'}, - namespace_packages=['wheezy'], + keywords="caching dependency memory null memcache memcached pylibmc", + packages=["wheezy", "wheezy.caching"], + package_dir={"": "src"}, + namespace_packages=["wheezy"], zip_safe=False, install_requires=[], extras_require={ - 'pylibmc': [ - 'pylibmc' - ], - 'python-memcached': [ - 'python-memcached' - ] + "pylibmc": ["pylibmc"], + "python-memcached": ["python-memcached"], }, - platforms='any', + platforms="any", **extra ) diff --git a/src/wheezy/__init__.py b/src/wheezy/__init__.py index 115ab1b..4dcc981 100644 --- a/src/wheezy/__init__.py +++ b/src/wheezy/__init__.py @@ -1,7 +1,8 @@ # See # http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages try: - __import__('pkg_resources').declare_namespace(__name__) + __import__("pkg_resources").declare_namespace(__name__) except ImportError: from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/src/wheezy/caching/__init__.py b/src/wheezy/caching/__init__.py index 26c2281..68cac2a 100644 --- a/src/wheezy/caching/__init__.py +++ b/src/wheezy/caching/__init__.py @@ -1,13 +1,7 @@ - """ """ # flake8: noqa -from wheezy.caching.client import CacheClient -from wheezy.caching.dependency import CacheDependency -from wheezy.caching.memory import MemoryCache -from wheezy.caching.null import NullCache - -__version__ = '0.1' +__version__ = "0.1" diff --git a/src/wheezy/caching/client.py b/src/wheezy/caching/client.py index c1b680e..8c92fde 100644 --- a/src/wheezy/caching/client.py +++ b/src/wheezy/caching/client.py @@ -1,4 +1,3 @@ - """ ``client`` module. """ @@ -26,38 +25,33 @@ def set(self, key, value, time=0, namespace=None): in cache. """ namespace = namespace or self.default_namespace - return self.namespaces[namespace].set( - key, value, time, namespace) + return self.namespaces[namespace].set(key, value, time, namespace) def set_multi(self, mapping, time=0, namespace=None): """ Set multiple keys' values at once. """ namespace = namespace or self.default_namespace - return self.namespaces[namespace].set_multi( - mapping, time, namespace) + return self.namespaces[namespace].set_multi(mapping, time, namespace) def add(self, key, value, time=0, namespace=None): """ Sets a key's value, if and only if the item is not already. """ namespace = namespace or self.default_namespace - return self.namespaces[namespace].add( - key, value, time, namespace) + return self.namespaces[namespace].add(key, value, time, namespace) def add_multi(self, mapping, time=0, namespace=None): """ Adds multiple values at once, with no effect for keys already in cache. """ namespace = namespace or self.default_namespace - return self.namespaces[namespace].add_multi( - mapping, time, namespace) + return self.namespaces[namespace].add_multi(mapping, time, namespace) def replace(self, key, value, time=0, namespace=None): """ Replaces a key's value, failing if item isn't already. """ namespace = namespace or self.default_namespace - return self.namespaces[namespace].replace( - key, value, time, namespace) + return self.namespaces[namespace].replace(key, value, time, namespace) def replace_multi(self, mapping, time=0, namespace=None): """ Replaces multiple values at once, with no effect for @@ -65,7 +59,8 @@ def replace_multi(self, mapping, time=0, namespace=None): """ namespace = namespace or self.default_namespace return self.namespaces[namespace].replace_multi( - mapping, time, namespace) + mapping, time, namespace + ) def get(self, key, namespace=None): """ Looks up a single key. @@ -78,8 +73,7 @@ def get_multi(self, keys, namespace=None): This is the recommended way to do bulk loads. """ namespace = namespace or self.default_namespace - return self.namespaces[namespace].get_multi( - keys, namespace) + return self.namespaces[namespace].get_multi(keys, namespace) def delete(self, key, seconds=0, namespace=None): """ Deletes a key from cache. @@ -92,7 +86,8 @@ def delete_multi(self, keys, seconds=0, namespace=None): """ namespace = namespace or self.default_namespace return self.namespaces[namespace].delete_multi( - keys, seconds, namespace) + keys, seconds, namespace + ) def incr(self, key, delta=1, namespace=None, initial_value=None): """ Atomically increments a key's value. The value, if too @@ -106,7 +101,8 @@ def incr(self, key, delta=1, namespace=None, initial_value=None): """ namespace = namespace or self.default_namespace return self.namespaces[namespace].incr( - key, delta, namespace, initial_value) + key, delta, namespace, initial_value + ) def decr(self, key, delta=1, namespace=None, initial_value=None): """ Atomically decrements a key's value. The value, if too @@ -120,7 +116,8 @@ def decr(self, key, delta=1, namespace=None, initial_value=None): """ namespace = namespace or self.default_namespace return self.namespaces[namespace].decr( - key, delta, namespace, initial_value) + key, delta, namespace, initial_value + ) def flush_all(self): """ Deletes everything in cache. diff --git a/src/wheezy/caching/comp.py b/src/wheezy/caching/comp.py index 5e73272..336aace 100644 --- a/src/wheezy/caching/comp.py +++ b/src/wheezy/caching/comp.py @@ -1,10 +1,8 @@ - """ ``comp`` module. """ import sys - PY_MAJOR = sys.version_info[0] PY_MINOR = sys.version_info[1] PY2 = PY_MAJOR == 2 @@ -12,6 +10,7 @@ if PY3: # pragma: nocover + def iteritems(d): return d.items() @@ -23,7 +22,10 @@ def itervalues(d): def list_map(f, iter): return list(map(f, iter)) + + else: # pragma: nocover + def iteritems(d): return d.iteritems() # noqa: B301 @@ -50,6 +52,7 @@ def itervalues(d): def __import__(name, globals=None, locals=None, f=None): # noqa: N807 return __saved_import__(name, globals, locals, f, 0) + if PY3: # pragma: nocover from queue import Queue else: # pragma: nocover diff --git a/src/wheezy/caching/dependency.py b/src/wheezy/caching/dependency.py index 6e5f0a0..e3e31eb 100644 --- a/src/wheezy/caching/dependency.py +++ b/src/wheezy/caching/dependency.py @@ -1,9 +1,7 @@ - """ ``dependency`` module. """ -from wheezy.caching.comp import itervalues -from wheezy.caching.comp import xrange +from wheezy.caching.comp import itervalues, xrange from wheezy.caching.utils import total_seconds @@ -28,8 +26,9 @@ def next_key(self, master_key): *master_key* - a key used to track a number of issued dependencies. """ - return master_key + str(self.cache.incr( - master_key, 1, self.namespace, 0)) + return master_key + str( + self.cache.incr(master_key, 1, self.namespace, 0) + ) def next_keys(self, master_key, n): """ Returns *n* number of dependency keys. @@ -37,20 +36,21 @@ def next_keys(self, master_key, n): *master_key* - a key used to track a number of issued dependencies. """ last_id = self.cache.incr(master_key, n, self.namespace, 0) - return [master_key + str(i) - for i in xrange(last_id - n + 1, last_id + 1)] + return [ + master_key + str(i) for i in xrange(last_id - n + 1, last_id + 1) + ] def add(self, master_key, key): """ Adds a given *key* to dependency. """ - return self.cache.add(self.next_key(master_key), - key, self.time, self.namespace) + return self.cache.add( + self.next_key(master_key), key, self.time, self.namespace + ) def add_multi(self, master_key, keys): """ Adds several *keys* to dependency. """ - mapping = dict(zip( - self.next_keys(master_key, len(keys)), keys)) + mapping = dict(zip(self.next_keys(master_key, len(keys)), keys)) return self.cache.add_multi(mapping, self.time, self.namespace) def get_keys(self, master_key): @@ -60,8 +60,7 @@ def get_keys(self, master_key): if n is None: return [] keys = [master_key + str(i) for i in xrange(1, n + 1)] - keys.extend(itervalues(self.cache.get_multi( - keys, self.namespace))) + keys.extend(itervalues(self.cache.get_multi(keys, self.namespace))) keys.append(master_key) return keys @@ -71,10 +70,12 @@ def get_multi_keys(self, master_keys): numbers = self.cache.get_multi(master_keys, self.namespace) if not numbers: return [] - keys = [master_key + str(i) for master_key, n in numbers.items() - for i in xrange(1, n + 1)] - keys.extend(itervalues(self.cache.get_multi( - keys, self.namespace))) + keys = [ + master_key + str(i) + for master_key, n in numbers.items() + for i in xrange(1, n + 1) + ] + keys.extend(itervalues(self.cache.get_multi(keys, self.namespace))) keys.extend(master_keys) return keys diff --git a/src/wheezy/caching/encoding.py b/src/wheezy/caching/encoding.py index 528462e..459ad35 100644 --- a/src/wheezy/caching/encoding.py +++ b/src/wheezy/caching/encoding.py @@ -1,4 +1,3 @@ - """ ``encoding`` module. """ @@ -6,8 +5,7 @@ from wheezy.caching.comp import string_type - -BASE64_ALTCHARS = '-_'.encode('latin1') +BASE64_ALTCHARS = "-_".encode("latin1") def encode_keys(mapping, key_encode): @@ -36,7 +34,7 @@ def string_encode(key): """ Encodes ``key`` with UTF-8 encoding. """ if isinstance(key, string_type): - return key.encode('UTF-8') + return key.encode("UTF-8") else: return key # pragma: nocover @@ -49,7 +47,7 @@ def base64_encode(key): True """ if isinstance(key, string_type): - key = key.encode('UTF-8') + key = key.encode("UTF-8") return b64encode(key, BASE64_ALTCHARS) @@ -76,7 +74,8 @@ def hash_encode(hash_factory): def key_encode(key): h = hash_factory() if isinstance(key, string_type): - key = key.encode('UTF-8') + key = key.encode("UTF-8") h.update(key) return h.digest() + return key_encode diff --git a/src/wheezy/caching/lockout.py b/src/wheezy/caching/lockout.py index 26d1c5c..3831b78 100644 --- a/src/wheezy/caching/lockout.py +++ b/src/wheezy/caching/lockout.py @@ -1,4 +1,3 @@ - """ ``lockout`` module. """ @@ -11,8 +10,9 @@ class Locker(object): """ Used to define lockout terms. """ - def __init__(self, cache, forbid_action, namespace=None, - key_prefix='c', **terms): + def __init__( + self, cache, forbid_action, namespace=None, key_prefix="c", **terms + ): self.cache = cache self.forbid_action = forbid_action self.namespace = namespace @@ -24,24 +24,31 @@ def define(self, name, **terms): The `terms` keys must correspond to `known terms` of locker. """ if not terms: # pragma: nocover - warn('Locker: no terms', stacklevel=2) - key_prefix = '%s:%s:' % (self.key_prefix, name.replace(' ', '_')) + warn("Locker: no terms", stacklevel=2) + key_prefix = "%s:%s:" % (self.key_prefix, name.replace(" ", "_")) counters = [self.terms[t](**terms[t]) for t in terms] - return Lockout(name, counters, self.forbid_action, - self.cache, self.namespace, key_prefix) + return Lockout( + name, + counters, + self.forbid_action, + self.cache, + self.namespace, + key_prefix, + ) class NullLocker(object): """ Null locker implementation. """ - def __init__(self, cache, forbid_action, namespace=None, - key_prefix='c', **terms): + def __init__( + self, cache, forbid_action, namespace=None, key_prefix="c", **terms + ): pass def define(self, name, **terms): if not terms: # pragma: nocover - warn('NullLocker: no terms', stacklevel=2) + warn("NullLocker: no terms", stacklevel=2) return NullLockout() @@ -49,8 +56,9 @@ class Counter(object): """ A container of various attributes used by lockout. """ - def __init__(self, key_func, count, period, duration, - reset=True, alert=None): + def __init__( + self, key_func, count, period, duration, reset=True, alert=None + ): self.key_func = key_func self.count = count self.period = total_seconds(period) @@ -63,8 +71,9 @@ class Lockout(object): """ A lockout is used to enforce terms of use policy. """ - def __init__(self, name, counters, forbid_action, - cache, namespace, key_prefix): + def __init__( + self, name, counters, forbid_action, cache, namespace, key_prefix + ): self.name = name self.counters = counters self.cache = cache @@ -78,6 +87,7 @@ def guard(self, func): subject to increase counter. The counters that support `reset` (and related locks) are deleted on success. """ + def guard_wrapper(ctx, *args, **kwargs): succeed = func(ctx, *args, **kwargs) if succeed: @@ -85,6 +95,7 @@ def guard_wrapper(ctx, *args, **kwargs): else: self.incr(ctx) return succeed + return guard_wrapper def quota(self, func): @@ -92,11 +103,13 @@ def quota(self, func): boolean indicating success or failure. Each success is a subject to increase counter. """ + def quota_wrapper(ctx, *args, **kwargs): succeed = func(ctx, *args, **kwargs) if succeed: self.incr(ctx) return succeed + return quota_wrapper def forbid_locked(self, wrapped=None, action=None): @@ -111,16 +124,19 @@ def forbid_locked(self, wrapped=None, action=None): assert action def decorate(func): - key_prefix = 'lock:' + self.key_prefix + key_prefix = "lock:" + self.key_prefix def forbid_locked_wrapper(ctx, *args, **kwargs): locks = self.cache.get_multi( [key_prefix + c.key_func(ctx) for c in self.counters], - self.namespace) + self.namespace, + ) if locks: return action(ctx) return func(ctx, *args, **kwargs) + return forbid_locked_wrapper + if wrapped is None: return decorate else: @@ -130,9 +146,8 @@ def reset(self, ctx): """ Removes locks for counters that support reset. """ key_prefix = self.key_prefix - keys = [key_prefix + c.key_func(ctx) - for c in self.counters if c.reset] - keys.extend(['lock:' + key for key in keys]) + keys = [key_prefix + c.key_func(ctx) for c in self.counters if c.reset] + keys.extend(["lock:" + key for key in keys]) keys and self.cache.delete_multi(keys, 0, self.namespace) def force_reset(self, ctx): @@ -140,7 +155,7 @@ def force_reset(self, ctx): """ key_prefix = self.key_prefix keys = [key_prefix + c.key_func(ctx) for c in self.counters] - keys.extend(['lock:' + key for key in keys]) + keys.extend(["lock:" + key for key in keys]) keys and self.cache.delete_multi(keys, 0, self.namespace) def incr(self, ctx): @@ -149,14 +164,15 @@ def incr(self, ctx): key_prefix = self.key_prefix for c in self.counters: key = key_prefix + c.key_func(ctx) - max_try = self.cache.add( - key, 1, c.period, self.namespace - ) and 1 or self.cache.incr(key, 1, self.namespace) + max_try = ( + self.cache.add(key, 1, c.period, self.namespace) + and 1 + or self.cache.incr(key, 1, self.namespace) + ) # print("%s ~ %d" % (key, max_try)) if max_try >= c.count: self.cache.delete(key, 0, self.namespace) - self.cache.add('lock:' + key, 1, - c.duration, self.namespace) + self.cache.add("lock:" + key, 1, c.duration, self.namespace) c.alert and c.alert(ctx, self.name, c) @@ -171,9 +187,9 @@ def quota(self, func): return func def forbid_locked(self, wrapped=None, action=None): - def decorate(func): return func + if wrapped is None: return decorate else: diff --git a/src/wheezy/caching/logging.py b/src/wheezy/caching/logging.py index 58e928d..7d559ac 100644 --- a/src/wheezy/caching/logging.py +++ b/src/wheezy/caching/logging.py @@ -1,4 +1,3 @@ - """ `logging` module. """ @@ -8,8 +7,7 @@ from wheezy.caching.encoding import hash_encode from wheezy.caching.utils import total_seconds - -Handler = __import__('logging', None, None, ['Handler']).Handler +Handler = __import__("logging", None, None, ["Handler"]).Handler class OnePassHandler(Handler): diff --git a/src/wheezy/caching/memcache.py b/src/wheezy/caching/memcache.py index 4c35e44..bca8272 100644 --- a/src/wheezy/caching/memcache.py +++ b/src/wheezy/caching/memcache.py @@ -1,17 +1,15 @@ - """ ``memcache`` module. """ -from wheezy.caching.comp import __import__ -from wheezy.caching.comp import list_map -from wheezy.caching.encoding import encode_keys -from wheezy.caching.encoding import string_encode +from wheezy.caching.comp import __import__, list_map +from wheezy.caching.encoding import encode_keys, string_encode try: - Client = __import__('memcache', None, None, ['Client']).Client + Client = __import__("memcache", None, None, ["Client"]).Client except ImportError: # pragma: nocover Client = None import warnings + warnings.warn("No module named 'memcache'", stacklevel=2) @@ -21,7 +19,7 @@ class MemcachedClient(object): """ def __init__(self, *args, **kwargs): - self.key_encode = kwargs.pop('key_encode', string_encode) + self.key_encode = kwargs.pop("key_encode", string_encode) if Client is None: # pragma: nocover raise ImportError("No module named 'memcache'") self.client = Client(*args, **kwargs) @@ -100,8 +98,9 @@ def delete(self, key, seconds=0, namespace=None): def delete_multi(self, keys, seconds=0, namespace=None): """ Delete multiple keys at once. """ - return self.client.delete_multi(map(self.key_encode, keys), - seconds) == 1 + return ( + self.client.delete_multi(map(self.key_encode, keys), seconds) == 1 + ) def incr(self, key, delta=1, namespace=None, initial_value=None): """ Atomically increments a key's value. The value, if too diff --git a/src/wheezy/caching/memory.py b/src/wheezy/caching/memory.py index edab7da..389ec3d 100644 --- a/src/wheezy/caching/memory.py +++ b/src/wheezy/caching/memory.py @@ -1,12 +1,9 @@ - """ ``memory`` module. """ from time import time as unixtime -from wheezy.caching.comp import allocate_lock -from wheezy.caching.comp import iteritems -from wheezy.caching.comp import xrange +from wheezy.caching.comp import allocate_lock, iteritems, xrange def expires(now, time): @@ -68,7 +65,8 @@ def find_expired(bucket_items, now): class CacheItem(object): """ A single cache item stored in cache. """ - __slots__ = ('key', 'value', 'expires') + + __slots__ = ("key", "value", "expires") def __init__(self, key, value, expires): self.key = key @@ -86,7 +84,8 @@ def __init__(self, buckets=60, bucket_interval=15): self.items = {} self.lock = allocate_lock() self.expire_buckets = [ - (allocate_lock(), []) for i in xrange(0, buckets)] + (allocate_lock(), []) for i in xrange(0, buckets) + ] self.last_expire_bucket_id = -1 def set(self, key, value, time=0, namespace=None): @@ -333,7 +332,8 @@ def incr(self, key, delta=1, namespace=None, initial_value=None): return None else: entry = items[key] = CacheItem( - key, initial_value, expires(now, 0)) + key, initial_value, expires(now, 0) + ) value = entry.value = entry.value + delta return value finally: @@ -489,6 +489,7 @@ def flush_all(self): return True -if __name__ == '__main__': # pragma: nocover +if __name__ == "__main__": # pragma: nocover import doctest + doctest.testmod() diff --git a/src/wheezy/caching/null.py b/src/wheezy/caching/null.py index a9e1175..460ba34 100644 --- a/src/wheezy/caching/null.py +++ b/src/wheezy/caching/null.py @@ -1,4 +1,3 @@ - """ ``interface`` module. """ diff --git a/src/wheezy/caching/patterns.py b/src/wheezy/caching/patterns.py index 37e2ae8..a54c34f 100644 --- a/src/wheezy/caching/patterns.py +++ b/src/wheezy/caching/patterns.py @@ -1,10 +1,8 @@ - """ ``patterns`` module. """ from inspect import getargspec -from time import sleep -from time import time +from time import sleep, time from wheezy.caching.dependency import CacheDependency from wheezy.caching.utils import total_seconds @@ -15,9 +13,15 @@ class Cached(object): for various cache operations and patterns. """ - def __init__(self, cache, key_builder=None, - time=0, namespace=None, - timeout=10, key_prefix='one_pass:'): + def __init__( + self, + cache, + key_builder=None, + time=0, + namespace=None, + timeout=10, + key_prefix="one_pass:", + ): self.cache = cache self.key_builder = key_builder self.time = total_seconds(time) @@ -140,7 +144,9 @@ def get_or_add_wrapper(*args, **kwargs): if result is not None: self.cache.add(key, result, self.time, self.namespace) return result + return get_or_add_wrapper + if wrapped is None: return decorate else: @@ -193,7 +199,9 @@ def get_or_set_wrapper(*args, **kwargs): if result is not None: self.cache.set(key, result, self.time, self.namespace) return result + return get_or_set_wrapper + if wrapped is None: return decorate else: @@ -212,22 +220,29 @@ def get_or_set_multi(self, make_key, create_factory, args): data_result = create_factory(args) elif len(cache_result) != len(key_map): data_result = create_factory( - [key_map[key] for key in key_map - if key not in cache_result]) + [key_map[key] for key in key_map if key not in cache_result] + ) else: - return dict([(key_map[key], cache_result[key]) - for key in cache_result]) + return dict( + [(key_map[key], cache_result[key]) for key in cache_result] + ) if not data_result: - return dict([(key_map[key], cache_result[key]) - for key in cache_result]) - self.set_multi(dict([ - (key, data_result[k]) - for key, k in key_map.items() - if k in data_result - ])) - data_result.update([(key_map[key], cache_result[key]) - for key in cache_result]) + return dict( + [(key_map[key], cache_result[key]) for key in cache_result] + ) + self.set_multi( + dict( + [ + (key, data_result[k]) + for key, k in key_map.items() + if k in data_result + ] + ) + ) + data_result.update( + [(key_map[key], cache_result[key]) for key in cache_result] + ) return data_result def wraps_get_or_set_multi(self, make_key): @@ -247,25 +262,28 @@ def get_multi_account(account_ids): def decorate(func): argnames = getargspec(func)[0] - if argnames and argnames[0] in ('self', 'cls', 'klass'): + if argnames and argnames[0] in ("self", "cls", "klass"): assert len(argnames) == 2 def get_or_set_multi_wrapper_with_ctx(ctx, args): return self.get_or_set_multi( - make_key, - lambda fargs: func(ctx, fargs), - args) + make_key, lambda fargs: func(ctx, fargs), args + ) + return get_or_set_multi_wrapper_with_ctx else: assert len(argnames) == 1 def get_or_set_multi_wrapper(args): return self.get_or_set_multi(make_key, func, args) + return get_or_set_multi_wrapper + return decorate - def one_pass_create(self, key, create_factory, - dependency_key_factory=None): + def one_pass_create( + self, key, create_factory, dependency_key_factory=None + ): """ Cache Pattern: try enter one pass: (1) if entered use *create_factory* to get a value if result is not `None` use cache `set` operation to store result and use @@ -275,8 +293,9 @@ def one_pass_create(self, key, create_factory, *cache*. """ result = None - one_pass = OnePass(self.cache, self.key_prefix + key, - self.timeout, self.namespace) + one_pass = OnePass( + self.cache, self.key_prefix + key, self.timeout, self.namespace + ) try: one_pass.__enter__() if one_pass.acquired: @@ -298,8 +317,9 @@ def get_or_create(self, key, create_factory, dependency_key_factory=None): result = self.cache.get(key, self.namespace) if result is not None: return result - return self.one_pass_create(key, create_factory, - dependency_key_factory) + return self.one_pass_create( + key, create_factory, dependency_key_factory + ) def wraps_get_or_create(self, wrapped=None, make_key=None): """ Returns specialized decorator for `get_or_create` cache @@ -314,6 +334,7 @@ def wraps_get_or_create(self, wrapped=None, make_key=None): def list_items(self, locale): pass """ + def decorate(func): mk = self.adapt(func, make_key) @@ -322,10 +343,10 @@ def get_or_create_wrapper(*args, **kwargs): result = self.cache.get(key, self.namespace) if result is not None: return result - return self.one_pass_create( - key, - lambda: func(*args, **kwargs)) + return self.one_pass_create(key, lambda: func(*args, **kwargs)) + return get_or_create_wrapper + if wrapped is None: return decorate else: @@ -336,9 +357,10 @@ def get_or_create_wrapper(*args, **kwargs): def adapt(self, func, make_key=None): if make_key: argnames = getargspec(func)[0] - if argnames and argnames[0] in ('self', 'cls', 'klass'): + if argnames and argnames[0] in ("self", "cls", "klass"): return lambda ignore, *args, **kwargs: make_key( - *args, **kwargs) + *args, **kwargs + ) else: return make_key else: @@ -361,7 +383,7 @@ class OnePass(object): # timeout """ - __slots__ = ('cache', 'key', 'time', 'namespace', 'acquired') + __slots__ = ("cache", "key", "time", "namespace", "acquired") def __init__(self, cache, key, time=10, namespace=None): self.cache = cache @@ -372,8 +394,9 @@ def __init__(self, cache, key, time=10, namespace=None): def __enter__(self): marker = int(time()) - self.acquired = self.cache.add(self.key, marker, self.time, - self.namespace) + self.acquired = self.cache.add( + self.key, marker, self.time, self.namespace + ) return self def wait(self, timeout=None): @@ -412,9 +435,9 @@ def key_format(func, key_prefix): """ argnames = getargspec(func)[0] n = len(argnames) - if n and argnames[0] in ('self', 'cls', 'klass'): + if n and argnames[0] in ("self", "cls", "klass"): n -= 1 - return '%s-%s%s' % (key_prefix, func.__name__, ':%r' * n) + return "%s-%s%s" % (key_prefix, func.__name__, ":%r" * n) def key_formatter(key_prefix): @@ -426,12 +449,14 @@ def key_formatter(key_prefix): >>> repo_key_format(list_items) 'repo-list_items:%r:%r' """ + def key_format_wrapper(func): return key_format(func, key_prefix) + return key_format_wrapper -def key_builder(key_prefix=''): +def key_builder(key_prefix=""): """ Returns a key builder that allows build a make cache key function at runtime. @@ -453,6 +478,7 @@ def key_list_items(self, locale='en', sort_order=1): return "repo-list_items:%r:%r" % (locale, sort_order) """ + def build(func): argnames, varargs, kwargs, defaults = getargspec(func) if defaults: @@ -462,20 +488,25 @@ def build(func): args.extend('%s=defaults["%s"]' % (n, n) for n in argnames[-n:]) else: args = argnames - if argnames and argnames[0] in ('self', 'cls', 'klass'): + if argnames and argnames[0] in ("self", "cls", "klass"): argnames = argnames[1:] - fname = 'key_' + func.__name__ + fname = "key_" + func.__name__ code = 'def %s(%s): return "%s" %% (%s)\ndel defaults' % ( - fname, ', '.join(args), key_format(func, key_prefix), - ', '.join(argnames)) - return compile_source(code, 'keys_' + key_prefix, defaults)[fname] + fname, + ", ".join(args), + key_format(func, key_prefix), + ", ".join(argnames), + ) + return compile_source(code, "keys_" + key_prefix, defaults)[fname] + return build # region: internal details + def compile_source(source, name, defaults): - compiled = compile(source, name, 'exec') - local_vars = {'defaults': defaults} + compiled = compile(source, name, "exec") + local_vars = {"defaults": defaults} exec(compiled, {}, local_vars) return local_vars diff --git a/src/wheezy/caching/pylibmc.py b/src/wheezy/caching/pylibmc.py index 1ff5d0c..f0ef4d5 100644 --- a/src/wheezy/caching/pylibmc.py +++ b/src/wheezy/caching/pylibmc.py @@ -1,29 +1,27 @@ - """ ``pylibmc`` module. """ -from wheezy.caching.comp import __import__ -from wheezy.caching.comp import list_map -from wheezy.caching.encoding import encode_keys -from wheezy.caching.encoding import string_encode +from wheezy.caching.comp import __import__, list_map +from wheezy.caching.encoding import encode_keys, string_encode try: - c = __import__('pylibmc', None, None, ['Client', 'NotFound']) + c = __import__("pylibmc", None, None, ["Client", "NotFound"]) Client = c.Client NotFound = c.NotFound def client_factory(*args, **kwargs): """ Client factory for pylibmc. """ - kwargs.setdefault('binary', True) - behaviors = kwargs.setdefault('behaviors', {}) - behaviors.setdefault('tcp_nodelay', True) - behaviors.setdefault('ketama', True) + kwargs.setdefault("binary", True) + behaviors = kwargs.setdefault("behaviors", {}) + behaviors.setdefault("tcp_nodelay", True) + behaviors.setdefault("ketama", True) return Client(*args, **kwargs) del c except ImportError: # pragma: nocover import warnings + warnings.warn("No module named 'pylibmc'", stacklevel=2) @@ -32,8 +30,8 @@ class MemcachedClient(object): """ def __init__(self, pool, key_encode=None): - assert hasattr(pool, 'acquire') - assert hasattr(pool, 'get_back') + assert hasattr(pool, "acquire") + assert hasattr(pool, "get_back") self.pool = pool self.key_encode = key_encode or string_encode @@ -103,8 +101,7 @@ def replace_multi(self, mapping, time=0, namespace=None): """ key_encode = self.key_encode failed = [] - mapping = [(key, key_encode(key), mapping[key]) - for key in mapping] + mapping = [(key, key_encode(key), mapping[key]) for key in mapping] try: client = self.pool.acquire() for key, key_encoded, value in mapping: diff --git a/src/wheezy/caching/tests/test_cache.py b/src/wheezy/caching/tests/test_cache.py index edf775b..4b984d9 100644 --- a/src/wheezy/caching/tests/test_cache.py +++ b/src/wheezy/caching/tests/test_cache.py @@ -1,4 +1,3 @@ - """ Unit tests for ``wheezy.caching``. """ @@ -6,7 +5,6 @@ class CacheTestMixin(object): - def setget(self, key, value): assert self.client.set(key, value, 10, self.namespace) is True assert value == self.client.get(key, self.namespace) @@ -19,74 +17,77 @@ def setget_multi(self, mapping): assert mapping[key] == values[key] def test_get_notfound(self): - assert self.client.get('uknown', self.namespace) is None + assert self.client.get("uknown", self.namespace) is None def test_get_multi_some_found(self): - self.setget('s1', 'some string') - assert {'s1': 'some string'} == self.client.get_multi( - ['unknown1', 's1'], namespace=self.namespace) + self.setget("s1", "some string") + assert {"s1": "some string"} == self.client.get_multi( + ["unknown1", "s1"], namespace=self.namespace + ) def test_get_multi_notfound(self): assert {} == self.client.get_multi( - ['unknown1', 'unknown2'], namespace=self.namespace) + ["unknown1", "unknown2"], namespace=self.namespace + ) def test_getset(self): - self.setget('s1', 'some string') - self.setget('i1', 100) + self.setget("s1", "some string") + self.setget("i1", 100) def test_getset_unicode_keys(self): - self.setget(string_type('s1'), 'some string') - self.setget(string_type('i1'), 100) + self.setget(string_type("s1"), "some string") + self.setget(string_type("i1"), 100) def test_getset_multi(self): - self.setget_multi({'k1': 'v1', 'k2': 'v2'}) + self.setget_multi({"k1": "v1", "k2": "v2"}) def test_getset_multi_unicode_keys(self): - self.setget_multi({ - string_type('k1'): 'v1', - string_type('k2'): 'v2' - }) + self.setget_multi({string_type("k1"): "v1", string_type("k2"): "v2"}) def test_add(self): - assert self.client.add('a', 100, namespace=self.namespace) - assert 100 == self.client.get('a', namespace=self.namespace) - assert not self.client.add('a', 100, namespace=self.namespace) + assert self.client.add("a", 100, namespace=self.namespace) + assert 100 == self.client.get("a", namespace=self.namespace) + assert not self.client.add("a", 100, namespace=self.namespace) def test_add_multi(self): - mapping = {'a1': 1, 'a2': 2} + mapping = {"a1": 1, "a2": 2} assert self.client.add_multi(mapping, namespace=self.namespace) == [] assert mapping == self.client.get_multi( - ['a1', 'a2'], namespace=self.namespace) - assert ['a1', 'a2'] == sorted(self.client.add_multi( - mapping, namespace=self.namespace)) + ["a1", "a2"], namespace=self.namespace + ) + assert ["a1", "a2"] == sorted( + self.client.add_multi(mapping, namespace=self.namespace) + ) def test_replace(self): - assert self.client.add('r', 100, namespace=self.namespace) - assert 100 == self.client.get('r', namespace=self.namespace) - assert self.client.replace('r', 101, namespace=self.namespace) - assert 101 == self.client.get('r', namespace=self.namespace) - assert not self.client.replace('rr', 101, namespace=self.namespace) + assert self.client.add("r", 100, namespace=self.namespace) + assert 100 == self.client.get("r", namespace=self.namespace) + assert self.client.replace("r", 101, namespace=self.namespace) + assert 101 == self.client.get("r", namespace=self.namespace) + assert not self.client.replace("rr", 101, namespace=self.namespace) def test_replace_multi(self): - mapping = {'r1': 1, 'r2': 2} - assert ['r1', 'r2'] == sorted(self.client.replace_multi( - mapping, namespace=self.namespace)) - assert [] == self.client.add_multi( - mapping, namespace=self.namespace) - mapping = {'r1': 100, 'r2': 200} + mapping = {"r1": 1, "r2": 2} + assert ["r1", "r2"] == sorted( + self.client.replace_multi(mapping, namespace=self.namespace) + ) + assert [] == self.client.add_multi(mapping, namespace=self.namespace) + mapping = {"r1": 100, "r2": 200} assert [] == self.client.replace_multi( - mapping, namespace=self.namespace) + mapping, namespace=self.namespace + ) assert mapping == self.client.get_multi( - ['r1', 'r2'], namespace=self.namespace) + ["r1", "r2"], namespace=self.namespace + ) def test_delete(self): - assert not self.client.delete('d', namespace=self.namespace) - self.setget('d', 1) - assert self.client.delete('d', namespace=self.namespace) - assert not self.client.get('d', self.namespace) + assert not self.client.delete("d", namespace=self.namespace) + self.setget("d", 1) + assert self.client.delete("d", namespace=self.namespace) + assert not self.client.get("d", self.namespace) def test_delete_multi(self): - mapping = {'d1': 1, 'd2': 2} + mapping = {"d1": 1, "d2": 2} keys = list(mapping.keys()) assert self.client.delete_multi(keys, namespace=self.namespace) self.setget_multi(mapping) @@ -95,25 +96,27 @@ def test_delete_multi(self): def test_incr(self): assert 1 == self.client.incr( - 'ci', namespace=self.namespace, initial_value=0) - assert 1 == self.client.get('ci', namespace=self.namespace) - assert 2 == self.client.incr('ci', namespace=self.namespace) - assert 2 == self.client.get('ci', namespace=self.namespace) + "ci", namespace=self.namespace, initial_value=0 + ) + assert 1 == self.client.get("ci", namespace=self.namespace) + assert 2 == self.client.incr("ci", namespace=self.namespace) + assert 2 == self.client.get("ci", namespace=self.namespace) def test_incr_returns_none(self): - assert self.client.incr('ix', namespace=self.namespace) is None + assert self.client.incr("ix", namespace=self.namespace) is None def test_decr(self): assert 9 == self.client.decr( - 'cd', namespace=self.namespace, initial_value=10) - assert 9 == self.client.get('cd', namespace=self.namespace) - assert 8 == self.client.decr('cd', namespace=self.namespace) - assert 8 == self.client.get('cd', namespace=self.namespace) + "cd", namespace=self.namespace, initial_value=10 + ) + assert 9 == self.client.get("cd", namespace=self.namespace) + assert 8 == self.client.decr("cd", namespace=self.namespace) + assert 8 == self.client.get("cd", namespace=self.namespace) def test_decr_none(self): - assert self.client.decr('dx') is None + assert self.client.decr("dx") is None def test_flush_all(self): - mapping = {'s1': 1, 's2': 2} + mapping = {"s1": 1, "s2": 2} assert [] == self.client.set_multi(mapping, namespace=self.namespace) assert self.client.flush_all() diff --git a/src/wheezy/caching/tests/test_client.py b/src/wheezy/caching/tests/test_client.py index 457440f..9fdfcf8 100644 --- a/src/wheezy/caching/tests/test_client.py +++ b/src/wheezy/caching/tests/test_client.py @@ -1,4 +1,3 @@ - """ """ @@ -8,19 +7,14 @@ from wheezy.caching.memory import MemoryCache from wheezy.caching.tests.test_cache import CacheTestMixin - cache1 = MemoryCache() cache2 = MemoryCache() client = CacheClient( - namespaces={ - 'cache1': cache1, - 'cache2': cache2 - }, - default_namespace='cache1') + namespaces={"cache1": cache1, "cache2": cache2}, default_namespace="cache1" +) class CacheClientDefaultTestCase(TestCase, CacheTestMixin): - def setUp(self): self.client = client self.namespace = None @@ -30,20 +24,18 @@ def tearDown(self): class CacheClientDefaultByNameTestCase(TestCase, CacheTestMixin): - def setUp(self): self.client = client - self.namespace = 'cache1' + self.namespace = "cache1" def tearDown(self): self.client.flush_all() class CacheClientByNamespaceTestCase(TestCase, CacheTestMixin): - def setUp(self): self.client = client - self.namespace = 'cache2' + self.namespace = "cache2" def tearDown(self): self.client.flush_all() diff --git a/src/wheezy/caching/tests/test_dependency.py b/src/wheezy/caching/tests/test_dependency.py index beb5ca7..f1c27dc 100644 --- a/src/wheezy/caching/tests/test_dependency.py +++ b/src/wheezy/caching/tests/test_dependency.py @@ -1,61 +1,59 @@ - """ Unit tests for ``wheezy.caching.patterns``. """ import unittest -from mock import ANY -from mock import Mock +from mock import ANY, Mock class CacheDependencyTestCase(unittest.TestCase): - def setUp(self): from wheezy.caching.dependency import CacheDependency + self.mock_cache = Mock() - self.d = CacheDependency(self.mock_cache, time=10, namespace='ns') + self.d = CacheDependency(self.mock_cache, time=10, namespace="ns") def test_next_key(self): """ Ensures consistency of key issued. """ self.mock_cache.incr.return_value = 1 - assert 'key1' == self.d.next_key('key') - self.mock_cache.incr.assert_called_once_with('key', 1, 'ns', 0) + assert "key1" == self.d.next_key("key") + self.mock_cache.incr.assert_called_once_with("key", 1, "ns", 0) self.mock_cache.reset_mock() self.mock_cache.incr.return_value = 2 - assert 'key2' == self.d.next_key('key') - self.mock_cache.incr.assert_called_once_with('key', 1, 'ns', 0) + assert "key2" == self.d.next_key("key") + self.mock_cache.incr.assert_called_once_with("key", 1, "ns", 0) def test_next_keys(self): """ Ensures consistency of keys issued. """ self.mock_cache.incr.return_value = 2 - assert ['key1', 'key2'] == self.d.next_keys('key', 2) - self.mock_cache.incr.assert_called_once_with('key', 2, 'ns', 0) + assert ["key1", "key2"] == self.d.next_keys("key", 2) + self.mock_cache.incr.assert_called_once_with("key", 2, "ns", 0) self.mock_cache.reset_mock() self.mock_cache.incr.return_value = 4 - assert ['key3', 'key4'] == self.d.next_keys('key', 2) - self.mock_cache.incr.assert_called_once_with('key', 2, 'ns', 0) + assert ["key3", "key4"] == self.d.next_keys("key", 2) + self.mock_cache.incr.assert_called_once_with("key", 2, "ns", 0) def test_add(self): """ Ensure dependency is added. """ self.mock_cache.incr.return_value = 1 self.mock_cache.add.return_value = True - assert self.d.add('key', 'k') - self.mock_cache.add.assert_called_once_with( - 'key1', 'k', 10, 'ns') + assert self.d.add("key", "k") + self.mock_cache.add.assert_called_once_with("key1", "k", 10, "ns") def test_add_multi(self): """ Ensure multiple dependency keys added. """ self.mock_cache.incr.return_value = 1 self.mock_cache.add.return_value = True - assert self.d.add_multi('key', ['k']) + assert self.d.add_multi("key", ["k"]) self.mock_cache.add_multi.assert_called_once_with( - {'key1': 'k'}, 10, 'ns') + {"key1": "k"}, 10, "ns" + ) self.mock_cache.reset_mock() self.mock_cache.incr.return_value = 1 @@ -65,97 +63,108 @@ def test_get_keys(self): """ Ensure related keys are returned. """ self.mock_cache.get.return_value = None - assert [] == self.d.get_keys('key') - self.mock_cache.get.assert_called_once_with( - 'key', 'ns') + assert [] == self.d.get_keys("key") + self.mock_cache.get.assert_called_once_with("key", "ns") assert not self.mock_cache.get_multi.called def side_effect(*args): - assert ['key1', 'key2'] == args[0] - return {'key1': 'k1', 'key2': 'k2'} + assert ["key1", "key2"] == args[0] + return {"key1": "k1", "key2": "k2"} + self.mock_cache.reset_mock() self.mock_cache.get.return_value = 2 self.mock_cache.get_multi.side_effect = side_effect - keys = self.d.get_keys('key') - assert ['k1', 'k2', 'key', 'key1', 'key2'] == sorted(keys) - self.mock_cache.get.assert_called_once_with( - 'key', 'ns') - self.mock_cache.get_multi.assert_called_once_with( - ANY, 'ns') + keys = self.d.get_keys("key") + assert ["k1", "k2", "key", "key1", "key2"] == sorted(keys) + self.mock_cache.get.assert_called_once_with("key", "ns") + self.mock_cache.get_multi.assert_called_once_with(ANY, "ns") def test_get_multi(self): """ Ensure related keys are returned for multi dependencies. """ self.mock_cache.get_multi.return_value = None - assert [] == self.d.get_multi_keys(['ka', 'kb']) - self.mock_cache.get_multi.assert_called_once_with( - ['ka', 'kb'], 'ns') + assert [] == self.d.get_multi_keys(["ka", "kb"]) + self.mock_cache.get_multi.assert_called_once_with(["ka", "kb"], "ns") - calls = [ - {'ka': 2, 'kb': 1}, - {'ka1': 'k1', 'ka2': 'k2', 'kb1': 'k3'} - ] + calls = [{"ka": 2, "kb": 1}, {"ka1": "k1", "ka2": "k2", "kb1": "k3"}] def side_effect(*args): result = calls[0] del calls[0] return result + self.mock_cache.reset_mock() self.mock_cache.get_multi.side_effect = side_effect - keys = self.d.get_multi_keys(['ka', 'kb', 'kc']) - assert ['k1', 'k2', 'k3', 'ka', 'ka1', 'ka2', 'kb', 'kb1', 'kc' - ] == sorted(keys) + keys = self.d.get_multi_keys(["ka", "kb", "kc"]) + assert [ + "k1", + "k2", + "k3", + "ka", + "ka1", + "ka2", + "kb", + "kb1", + "kc", + ] == sorted(keys) assert 2 == self.mock_cache.get_multi.call_count def test_delete(self): """ Ensure related keys are invalidated. """ self.mock_cache.get.return_value = None - assert self.d.delete('key') - self.mock_cache.get.assert_called_once_with('key', 'ns') + assert self.d.delete("key") + self.mock_cache.get.assert_called_once_with("key", "ns") assert not self.mock_cache.get_multi.called assert not self.mock_cache.delete_multi.called def side_effect(*args): - assert ['key1', 'key2'] == args[0] - return {'key1': 'k1', 'key2': 'k2'} + assert ["key1", "key2"] == args[0] + return {"key1": "k1", "key2": "k2"} + self.mock_cache.reset_mock() self.mock_cache.get.return_value = 2 self.mock_cache.get_multi.side_effect = side_effect - assert self.d.delete('key') - self.mock_cache.get.assert_called_once_with('key', 'ns') - self.mock_cache.get_multi.assert_called_once_with(ANY, 'ns') - self.mock_cache.delete_multi.assert_called_once_with(ANY, 0, 'ns') - assert ['k1', 'k2', 'key', 'key1', 'key2'] == sorted( - self.mock_cache.delete_multi.call_args[0][0]) + assert self.d.delete("key") + self.mock_cache.get.assert_called_once_with("key", "ns") + self.mock_cache.get_multi.assert_called_once_with(ANY, "ns") + self.mock_cache.delete_multi.assert_called_once_with(ANY, 0, "ns") + assert ["k1", "k2", "key", "key1", "key2"] == sorted( + self.mock_cache.delete_multi.call_args[0][0] + ) def test_delete_multi(self): """ Ensure related keys are invalidated for multi dependencies. """ self.mock_cache.get_multi.return_value = None - assert self.d.delete_multi(['ka', 'kb']) - self.mock_cache.get_multi.assert_called_once_with( - ['ka', 'kb'], 'ns') + assert self.d.delete_multi(["ka", "kb"]) + self.mock_cache.get_multi.assert_called_once_with(["ka", "kb"], "ns") assert not self.mock_cache.delete_multi.called - calls = [ - {'ka': 2, 'kb': 1}, - {'ka1': 'k1', 'ka2': 'k2', 'kb1': 'k3'} - ] + calls = [{"ka": 2, "kb": 1}, {"ka1": "k1", "ka2": "k2", "kb1": "k3"}] def side_effect(*args): result = calls[0] del calls[0] return result + self.mock_cache.reset_mock() self.mock_cache.get_multi.side_effect = side_effect - assert self.d.delete_multi(['ka', 'kb', 'kc']) + assert self.d.delete_multi(["ka", "kb", "kc"]) assert 2 == self.mock_cache.get_multi.call_count - self.mock_cache.delete_multi.assert_called_once_with( - ANY, 0, 'ns') - assert ['k1', 'k2', 'k3', 'ka', 'ka1', 'ka2', 'kb', 'kb1', 'kc' - ] == sorted(self.mock_cache.delete_multi.call_args[0][0]) + self.mock_cache.delete_multi.assert_called_once_with(ANY, 0, "ns") + assert [ + "k1", + "k2", + "k3", + "ka", + "ka1", + "ka2", + "kb", + "kb1", + "kc", + ] == sorted(self.mock_cache.delete_multi.call_args[0][0]) diff --git a/src/wheezy/caching/tests/test_lockout.py b/src/wheezy/caching/tests/test_lockout.py index fa181c5..1c0c2e6 100644 --- a/src/wheezy/caching/tests/test_lockout.py +++ b/src/wheezy/caching/tests/test_lockout.py @@ -1,15 +1,12 @@ - """ Unit tests for ``wheezy.caching.lockout``. """ import unittest from datetime import timedelta -from wheezy.caching.lockout import Counter -from wheezy.caching.lockout import Locker +from wheezy.caching.lockout import Counter, Locker from wheezy.caching.memory import MemoryCache - # region: alerts alerts = [] @@ -17,84 +14,110 @@ def send_mail(s, name, counter): assert isinstance(s, MyService) - alerts.append('send mail: %s' % name) + alerts.append("send mail: %s" % name) def send_sms(s, name, counter): assert isinstance(s, MyService) - alerts.append('send sms: %s' % name) + alerts.append("send sms: %s" % name) def ignore_alert(s, name, counter): assert isinstance(s, MyService) - alerts.append('ignore: %s' % name) + alerts.append("ignore: %s" % name) # region: lockouts and defaults -def lockout_by_id(count=10, - period=timedelta(minutes=1), # noqa: B008 - duration=timedelta(hours=2), # noqa: B008 - reset=False, - alert=send_mail): +def lockout_by_id( + count=10, + period=timedelta(minutes=1), # noqa: B008 + duration=timedelta(hours=2), # noqa: B008 + reset=False, + alert=send_mail, +): def key_func(s): - return 'by_id:%s' % s.user_id - return Counter(key_func=key_func, count=count, - period=period, duration=duration, - reset=reset, alert=alert) - + return "by_id:%s" % s.user_id + + return Counter( + key_func=key_func, + count=count, + period=period, + duration=duration, + reset=reset, + alert=alert, + ) -def lockout_by_ip(count=10, - period=timedelta(minutes=1), # noqa: B008 - duration=timedelta(hours=2), # noqa: B008 - reset=True, - alert=send_sms): +def lockout_by_ip( + count=10, + period=timedelta(minutes=1), # noqa: B008 + duration=timedelta(hours=2), # noqa: B008 + reset=True, + alert=send_sms, +): def key_func(s): - return 'by_ip:%s' % s.user_ip - return Counter(key_func=key_func, count=count, - period=period, duration=duration, - reset=reset, alert=alert) - + return "by_ip:%s" % s.user_ip + + return Counter( + key_func=key_func, + count=count, + period=period, + duration=duration, + reset=reset, + alert=alert, + ) -def lockout_by_id_ip(count=10, - period=timedelta(minutes=1), # noqa: B008 - duration=timedelta(hours=2), # noqa: B008 - reset=True, - alert=ignore_alert): +def lockout_by_id_ip( + count=10, + period=timedelta(minutes=1), # noqa: B008 + duration=timedelta(hours=2), # noqa: B008 + reset=True, + alert=ignore_alert, +): def key_func(s): - return 'by_id_ip:%s:%s' % (s.user_id, s.user_ip) - return Counter(key_func=key_func, count=count, - period=period, duration=duration, - reset=reset, alert=alert) + return "by_id_ip:%s:%s" % (s.user_id, s.user_ip) + + return Counter( + key_func=key_func, + count=count, + period=period, + duration=duration, + reset=reset, + alert=alert, + ) # region: config cache = MemoryCache() -locker = Locker(cache, key_prefix='my_app', - forbid_action=lambda s: 'forbidden', - by_id=lockout_by_id, - by_ip=lockout_by_ip, - by_id_ip=lockout_by_id_ip) +locker = Locker( + cache, + key_prefix="my_app", + forbid_action=lambda s: "forbidden", + by_id=lockout_by_id, + by_ip=lockout_by_ip, + by_id_ip=lockout_by_id_ip, +) # region: service/handler + class MyService(object): lockout = locker.define( - name='action', + name="action", by_id_ip=dict(count=4, duration=60), by_id=dict(count=6, duration=timedelta(minutes=2)), - by_ip=dict(count=8, duration=timedelta(minutes=5)) + by_ip=dict(count=8, duration=timedelta(minutes=5)), ) lockout2 = locker.define( - name='action 2', - by_ip=dict(count=4, duration=timedelta(minutes=5), reset=False) + name="action 2", + by_ip=dict(count=4, duration=timedelta(minutes=5), reset=False), ) action_result = False @@ -104,16 +127,16 @@ class MyService(object): @lockout.forbid_locked def action(self): if self.do_action(): - return 'show ok' + return "show ok" else: - return 'show error' + return "show error" - @lockout.forbid_locked(action=lambda s: 'show captcha') + @lockout.forbid_locked(action=lambda s: "show captcha") def action2(self): if self.do_action(): # pragma: nocover - return 'show ok' + return "show ok" else: - return 'show error' + return "show error" @lockout.guard def do_action(self): @@ -122,9 +145,9 @@ def do_action(self): @lockout.forbid_locked def action3(self): if self.do_action3(): - return 'show ok' + return "show ok" else: - return 'show error' + return "show error" @lockout.quota def do_action3(self): @@ -133,9 +156,9 @@ def do_action3(self): @lockout2.forbid_locked def action4(self): if self.do_action4(): # pragma: nocover - return 'show ok' + return "show ok" else: - return 'show error' + return "show error" @lockout2.guard def do_action4(self): @@ -144,129 +167,133 @@ def do_action4(self): # region: test case -class LockoutTestCase(unittest.TestCase): +class LockoutTestCase(unittest.TestCase): def setUp(self): del alerts[:] cache.flush_all() def test_forbidden(self): s = MyService() - s.user_id = 'u1' - s.user_ip = 'ip1' + s.user_id = "u1" + s.user_ip = "ip1" for _ in range(4): - assert 'show error' == s.action() - assert ['ignore: action'] == alerts + assert "show error" == s.action() + assert ["ignore: action"] == alerts del alerts[:] - assert 'forbidden' == s.action(), 'lock by id/ip' + assert "forbidden" == s.action(), "lock by id/ip" - s.user_ip = 'ip2' + s.user_ip = "ip2" for _ in range(2): - assert 'show error' == s.action() - assert ['send mail: action'] == alerts + assert "show error" == s.action() + assert ["send mail: action"] == alerts del alerts[:] - assert 'forbidden' == s.action(), 'lock by id' + assert "forbidden" == s.action(), "lock by id" - s.user_id = 'u3' + s.user_id = "u3" for _ in range(3): - assert 'show error' == s.action() - s.user_id = 'u4' + assert "show error" == s.action() + s.user_id = "u4" for _ in range(3): - assert 'show error' == s.action() - assert ['send sms: action'] == alerts - assert 'forbidden' == s.action(), 'lock by ip' + assert "show error" == s.action() + assert ["send sms: action"] == alerts + assert "forbidden" == s.action(), "lock by ip" def test_reset_on_success(self): s = MyService() - s.user_id = 'u0' - s.user_ip = 'ip0' + s.user_id = "u0" + s.user_ip = "ip0" for _ in range(2): - assert 'show error' == s.action() + assert "show error" == s.action() s.action_result = True - assert 'show ok' == s.action() + assert "show ok" == s.action() s.action_result = False for _ in range(4): - assert 'show error' == s.action() - assert 'forbidden' == s.action() + assert "show error" == s.action() + assert "forbidden" == s.action() def test_reset(self): s = MyService() - s.user_id = 'u0' - s.user_ip = 'ip0' + s.user_id = "u0" + s.user_ip = "ip0" # reset supported for _ in range(4): - assert 'show error' == s.action() - assert 'forbidden' == s.action() + assert "show error" == s.action() + assert "forbidden" == s.action() s.lockout.reset(s) - assert 'show error' == s.action() + assert "show error" == s.action() # reset not supported for _ in range(4): s.action4() - assert 'forbidden' == s.action4() + assert "forbidden" == s.action4() s.lockout2.reset(s) - assert 'forbidden' == s.action4() + assert "forbidden" == s.action4() def test_force_reset(self): s = MyService() - s.user_id = 'u0' - s.user_ip = 'ip0' + s.user_id = "u0" + s.user_ip = "ip0" # reset supported for _ in range(4): - assert 'show error' == s.action() - assert 'forbidden' == s.action() + assert "show error" == s.action() + assert "forbidden" == s.action() s.lockout.force_reset(s) - assert 'show error' == s.action() + assert "show error" == s.action() # reset not supported for _ in range(4): s.action4() - assert 'forbidden' == s.action4() + assert "forbidden" == s.action4() s.lockout2.force_reset(s) - assert 'show error' == s.action4() + assert "show error" == s.action4() def test_custom_forbid_action(self): s = MyService() - s.user_id = 'cfa-u1' - s.user_ip = 'cfa-ip1' + s.user_id = "cfa-u1" + s.user_ip = "cfa-ip1" for _ in range(4): - assert 'show error' == s.action2() - assert 'show captcha' == s.action2() + assert "show error" == s.action2() + assert "show captcha" == s.action2() def test_quota(self): s = MyService() - s.user_id = 'u1q' - s.user_ip = 'ip1q' + s.user_id = "u1q" + s.user_ip = "ip1q" for _ in range(10): - assert 'show error' == s.action3() + assert "show error" == s.action3() assert [] == alerts del alerts[:] s.action_result = True for _ in range(4): - assert 'show ok' == s.action3() - assert ['ignore: action'] == alerts - assert 'forbidden' == s.action3(), 'lock by id/ip' + assert "show ok" == s.action3() + assert ["ignore: action"] == alerts + assert "forbidden" == s.action3(), "lock by id/ip" class NullLockoutTestCase(unittest.TestCase): - def test_locker(self): - from wheezy.caching.lockout import NullLocker - from wheezy.caching.lockout import NullLockout - locker = NullLocker(cache, key_prefix='my_app', - forbid_action=lambda s: 'forbidden', - by_id=lockout_by_id, - by_ip=lockout_by_ip, - by_id_ip=lockout_by_id_ip) - assert isinstance(locker.define('x', by_id_ip={}), NullLockout) + from wheezy.caching.lockout import NullLocker, NullLockout + + locker = NullLocker( + cache, + key_prefix="my_app", + forbid_action=lambda s: "forbidden", + by_id=lockout_by_id, + by_ip=lockout_by_ip, + by_id_ip=lockout_by_id_ip, + ) + assert isinstance(locker.define("x", by_id_ip={}), NullLockout) def test_lockout(self): from wheezy.caching.lockout import NullLockout + lockout = NullLockout() def f(): pass # pragma: nocover + assert f == lockout.guard(f) assert f == lockout.quota(f) assert f == lockout.forbid_locked(f) diff --git a/src/wheezy/caching/tests/test_logging.py b/src/wheezy/caching/tests/test_logging.py index d915310..ca440de 100644 --- a/src/wheezy/caching/tests/test_logging.py +++ b/src/wheezy/caching/tests/test_logging.py @@ -1,4 +1,3 @@ - """ Unit tests for ``wheezy.caching.logging``. """ @@ -8,9 +7,9 @@ class OnePassHandlerTestCase(unittest.TestCase): - def setUp(self): from wheezy.caching.logging import OnePassHandler + self.mock_inner = Mock() self.mock_cache = Mock() self.h = OnePassHandler(self.mock_inner, self.mock_cache, 60) @@ -19,7 +18,7 @@ def test_emit_first_call(self): """ Ensure the inner handler emit on first call. """ mock_record = Mock() - mock_record.getMessage.return_value = 'msg' + mock_record.getMessage.return_value = "msg" self.mock_cache.add.return_value = True self.h.emit(mock_record) assert self.mock_cache.add.called @@ -29,7 +28,7 @@ def test_emit_next_call(self): """ Ensure there is no call to inner handler emit on next call. """ mock_record = Mock() - mock_record.getMessage.return_value = 'msg' + mock_record.getMessage.return_value = "msg" self.mock_cache.add.return_value = False self.h.emit(mock_record) assert self.mock_cache.add.called diff --git a/src/wheezy/caching/tests/test_memcache.py b/src/wheezy/caching/tests/test_memcache.py index 5642e91..16be80f 100644 --- a/src/wheezy/caching/tests/test_memcache.py +++ b/src/wheezy/caching/tests/test_memcache.py @@ -1,4 +1,3 @@ - """ Unit tests for ``wheezy.caching.memcache``. """ @@ -8,22 +7,20 @@ from wheezy.caching.memcache import Client from wheezy.caching.tests.test_cache import CacheTestMixin - if Client: from wheezy.caching.memcache import MemcachedClient class MemcacheClientTestCase(TestCase, CacheTestMixin): - def setUp(self): - self.client = MemcachedClient([ - os.environ.get('MEMCACHED_HOST', '127.0.0.1') - ]) + self.client = MemcachedClient( + [os.environ.get("MEMCACHED_HOST", "127.0.0.1")] + ) self.namespace = None def tearDown(self): self.client.flush_all() def test_delete(self): - assert self.client.delete('d') - self.setget('d', 1) - assert self.client.delete('d') + assert self.client.delete("d") + self.setget("d", 1) + assert self.client.delete("d") diff --git a/src/wheezy/caching/tests/test_memory.py b/src/wheezy/caching/tests/test_memory.py index e7b4113..ebe92e3 100644 --- a/src/wheezy/caching/tests/test_memory.py +++ b/src/wheezy/caching/tests/test_memory.py @@ -1,4 +1,3 @@ - """ Unit tests for ``wheezy.caching.memory``. """ @@ -9,7 +8,6 @@ class MemoryCacheTestCase(TestCase, CacheTestMixin): - def setUp(self): self.client = MemoryCache() self.namespace = None diff --git a/src/wheezy/caching/tests/test_patterns.py b/src/wheezy/caching/tests/test_patterns.py index dc6d9c3..24e6a81 100644 --- a/src/wheezy/caching/tests/test_patterns.py +++ b/src/wheezy/caching/tests/test_patterns.py @@ -1,130 +1,131 @@ - """ Unit tests for ``wheezy.caching.patterns``. """ import unittest -from mock import ANY -from mock import Mock -from mock import patch +from mock import ANY, Mock, patch class CachedTestCase(unittest.TestCase): - def setUp(self): from wheezy.caching.patterns import Cached + self.mock_cache = Mock() self.mock_dependency = Mock() - self.cached = Cached(self.mock_cache, time=10, namespace='ns') + self.cached = Cached(self.mock_cache, time=10, namespace="ns") self.cached.dependency = self.mock_dependency def test_set(self): """ Ensure set operation is passed to cache. """ - self.cached.set('key', 'value') - self.mock_cache.set.assert_called_once_with('key', 'value', 10, 'ns') + self.cached.set("key", "value") + self.mock_cache.set.assert_called_once_with("key", "value", 10, "ns") def test_set_with_dependency(self): """ Ensure set operation is passed to cache and key added to dependency. """ - self.cached.set('key', 'value', 'master_key') - self.mock_cache.set.assert_called_once_with('key', 'value', 10, 'ns') - self.mock_dependency.add.assert_called_once_with('master_key', 'key') + self.cached.set("key", "value", "master_key") + self.mock_cache.set.assert_called_once_with("key", "value", 10, "ns") + self.mock_dependency.add.assert_called_once_with("master_key", "key") def test_set_multi(self): """ Ensure set_multi operation is passed to cache. """ - self.cached.set_multi({'key': 'value'}) + self.cached.set_multi({"key": "value"}) self.mock_cache.set_multi.assert_called_once_with( - {'key': 'value'}, 10, 'ns') + {"key": "value"}, 10, "ns" + ) def test_add(self): """ Ensure add operation is passed to cache. """ - self.cached.add('key', 'value') - self.mock_cache.add.assert_called_once_with('key', 'value', 10, 'ns') + self.cached.add("key", "value") + self.mock_cache.add.assert_called_once_with("key", "value", 10, "ns") def test_add_failed_with_dependency(self): """ Ensure add operation is passed to cache and key added to dependency. """ self.mock_cache.add.return_value = False - self.cached.add('key', 'value', self.mock_dependency) - self.mock_cache.add.assert_called_once_with('key', 'value', 10, 'ns') + self.cached.add("key", "value", self.mock_dependency) + self.mock_cache.add.assert_called_once_with("key", "value", 10, "ns") assert not self.mock_dependency.add.called def test_add_with_dependency(self): """ Ensure add operation is passed to cache and key added to dependency. """ - self.cached.add('key', 'value', 'master_key') - self.mock_cache.add.assert_called_once_with('key', 'value', 10, 'ns') - self.mock_dependency.add.assert_called_once_with('master_key', 'key') + self.cached.add("key", "value", "master_key") + self.mock_cache.add.assert_called_once_with("key", "value", 10, "ns") + self.mock_dependency.add.assert_called_once_with("master_key", "key") def test_add_multi(self): """ Ensure add_multi operation is passed to cache. """ - self.cached.add_multi({'key': 'value'}) + self.cached.add_multi({"key": "value"}) self.mock_cache.add_multi.assert_called_once_with( - {'key': 'value'}, 10, 'ns') + {"key": "value"}, 10, "ns" + ) def test_replace(self): """ Ensure replace operation is passed to cache. """ - self.cached.replace('key', 'value') + self.cached.replace("key", "value") self.mock_cache.replace.assert_called_once_with( - 'key', 'value', 10, 'ns') + "key", "value", 10, "ns" + ) def test_replace_multi(self): """ Ensure replace_multi operation is passed to cache. """ - self.cached.replace_multi({'key': 'value'}) + self.cached.replace_multi({"key": "value"}) self.mock_cache.replace_multi.assert_called_once_with( - {'key': 'value'}, 10, 'ns') + {"key": "value"}, 10, "ns" + ) def test_get(self): """ Ensure get operation is passed to cache. """ - self.cached.get('key') - self.mock_cache.get.assert_called_once_with('key', 'ns') + self.cached.get("key") + self.mock_cache.get.assert_called_once_with("key", "ns") def test_get_multi(self): """ Ensure get_multi operation is passed to cache. """ - self.cached.get_multi(['key']) - self.mock_cache.get_multi.assert_called_once_with(['key'], 'ns') + self.cached.get_multi(["key"]) + self.mock_cache.get_multi.assert_called_once_with(["key"], "ns") def test_delete(self): """ Ensure delete operation is passed to cache. """ - self.cached.delete('key', 0) - self.mock_cache.delete.assert_called_once_with('key', 0, 'ns') + self.cached.delete("key", 0) + self.mock_cache.delete.assert_called_once_with("key", 0, "ns") def test_delete_multi(self): """ Ensure delete_multi operation is passed to cache. """ - self.cached.delete_multi(['key'], 0) - self.mock_cache.delete_multi.assert_called_once_with( - ['key'], 0, 'ns') + self.cached.delete_multi(["key"], 0) + self.mock_cache.delete_multi.assert_called_once_with(["key"], 0, "ns") def test_incr(self): """ Ensure incr operation is passed to cache. """ - self.cached.incr('key', 1, 0) - self.mock_cache.incr.assert_called_once_with('key', 1, 'ns', 0) + self.cached.incr("key", 1, 0) + self.mock_cache.incr.assert_called_once_with("key", 1, "ns", 0) def test_decr(self): """ Ensure decr operation is passed to cache. """ - self.cached.decr('key', 1, 0) - self.mock_cache.decr.assert_called_once_with('key', 1, 'ns', 0) + self.cached.decr("key", 1, 0) + self.mock_cache.decr.assert_called_once_with("key", 1, "ns", 0) def test_dependency(self): """ Ensure returned CacheDependency is properly initialized. """ from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns') + + cached = Cached(self.mock_cache, time=10, namespace="ns") d = cached.dependency assert cached.cache == d.cache assert cached.time == d.time @@ -135,33 +136,36 @@ def test_adapt_make_key(self): """ def make_key(): - return 'key' + return "key" def my_func(): pass # pragma: nocover + mk = self.cached.adapt(my_func, make_key) - assert 'key' == mk() + assert "key" == mk() def test_adapt_make_key_cls(self): """ Ignore 'cls' argument. """ def make_key(): - return 'key' + return "key" def my_func(cls): pass # pragma: nocover + mk = self.cached.adapt(my_func, make_key) - assert 'key' == mk('cls') + assert "key" == mk("cls") class OnePassTestCase(unittest.TestCase): - def setUp(self): from wheezy.caching.patterns import OnePass + self.mock_cache = Mock() - self.one_pass = OnePass(self.mock_cache, 'key', - time=10, namespace='ns') + self.one_pass = OnePass( + self.mock_cache, "key", time=10, namespace="ns" + ) def test_enter(self): """ Enter returns one_pass instance. @@ -169,7 +173,7 @@ def test_enter(self): self.mock_cache.add.return_value = True assert self.one_pass == self.one_pass.__enter__() assert self.one_pass.acquired - self.mock_cache.add.assert_called_once_with('key', ANY, 10, 'ns') + self.mock_cache.add.assert_called_once_with("key", ANY, 10, "ns") def test_exit_acquired(self): """ Releases key if acquired. @@ -178,7 +182,7 @@ def test_exit_acquired(self): self.one_pass.__enter__() assert self.one_pass.acquired self.one_pass.__exit__(None, None, None) - self.mock_cache.delete.assert_called_once_with('key', 0, 'ns') + self.mock_cache.delete.assert_called_once_with("key", 0, "ns") assert not self.one_pass.acquired def test_exit_not_acquired(self): @@ -190,7 +194,7 @@ def test_exit_not_acquired(self): self.one_pass.__exit__(None, None, None) assert not self.mock_cache.delete.called - @patch('wheezy.caching.patterns.sleep') + @patch("wheezy.caching.patterns.sleep") def test_wait_no_marker(self, mock_sleep): """ Exit wait loop if there is no marker. """ @@ -200,7 +204,7 @@ def test_wait_no_marker(self, mock_sleep): self.mock_cache.get.return_value = None assert self.one_pass.wait() - @patch('wheezy.caching.patterns.sleep') + @patch("wheezy.caching.patterns.sleep") def test_wait_timeout(self, mock_sleep): """ Exit wait loop if there is timeout reached. """ @@ -212,7 +216,6 @@ def test_wait_timeout(self, mock_sleep): class GetOrAddTestCase(unittest.TestCase): - def setUp(self): self.mock_cache = Mock() self.mock_create_factory = Mock() @@ -220,17 +223,19 @@ def setUp(self): def get_or_add(self, dependency_key_factory=None): from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns') + + cached = Cached(self.mock_cache, time=10, namespace="ns") cached.dependency = self.mock_dependency - return cached.get_or_add('key', self.mock_create_factory, - dependency_key_factory) + return cached.get_or_add( + "key", self.mock_create_factory, dependency_key_factory + ) def test_found(self): """ An item found in cache. """ - self.mock_cache.get.return_value = 'x' - assert 'x' == self.get_or_add() - self.mock_cache.get.assert_called_once_with('key', 'ns') + self.mock_cache.get.return_value = "x" + assert "x" == self.get_or_add() + self.mock_cache.get.assert_called_once_with("key", "ns") assert not self.mock_create_factory.called def test_create_none(self): @@ -240,7 +245,7 @@ def test_create_none(self): self.mock_create_factory.return_value = None assert not self.get_or_add() - self.mock_cache.get.assert_called_once_with('key', 'ns') + self.mock_cache.get.assert_called_once_with("key", "ns") self.mock_create_factory.assert_called_once_with() assert not self.mock_cache.add.called @@ -248,30 +253,29 @@ def test_add_failed(self): """ Attempt to add a value to cache failed. """ self.mock_cache.get.return_value = None - self.mock_create_factory.return_value = 'x' + self.mock_create_factory.return_value = "x" self.mock_cache.add.return_value = False - assert 'x' == self.get_or_add() - self.mock_cache.get.assert_called_once_with('key', 'ns') - self.mock_cache.add.assert_called_once_with('key', 'x', 10, 'ns') + assert "x" == self.get_or_add() + self.mock_cache.get.assert_called_once_with("key", "ns") + self.mock_cache.add.assert_called_once_with("key", "x", 10, "ns") def test_has_dependency(self): """ There is specified `dependency_factory`. """ self.mock_cache.get.return_value = None - self.mock_create_factory.return_value = 'x' + self.mock_create_factory.return_value = "x" self.mock_cache.add.return_value = True mock_dependency_key_factory = Mock() - mock_dependency_key_factory.return_value = 'master_key' + mock_dependency_key_factory.return_value = "master_key" - assert 'x' == self.get_or_add(mock_dependency_key_factory) - self.mock_cache.get.assert_called_once_with('key', 'ns') - self.mock_cache.add.assert_called_once_with('key', 'x', 10, 'ns') - self.mock_dependency.add.assert_called_once_with('master_key', 'key') + assert "x" == self.get_or_add(mock_dependency_key_factory) + self.mock_cache.get.assert_called_once_with("key", "ns") + self.mock_cache.add.assert_called_once_with("key", "x", 10, "ns") + self.mock_dependency.add.assert_called_once_with("master_key", "key") class WrapsGetOrAddTestCase(GetOrAddTestCase): - def test_has_dependency(self): pass @@ -280,26 +284,28 @@ def get_or_add(self, dependency_factory=None): def kb(f): def key(*args, **kwargs): - return 'key' + return "key" + return key - cached = Cached(self.mock_cache, kb, time=10, namespace='ns') + + cached = Cached(self.mock_cache, kb, time=10, namespace="ns") return cached.wraps_get_or_add(self.mock_create_factory)() class WrapsGetOrAddMakeKeyTestCase(WrapsGetOrAddTestCase): - def get_or_add(self, dependency_factory=None): from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns') - @cached.wraps_get_or_add(make_key=lambda: 'key') + cached = Cached(self.mock_cache, time=10, namespace="ns") + + @cached.wraps_get_or_add(make_key=lambda: "key") def create_factory(): return self.mock_create_factory() + return create_factory() class GetOrSetTestCase(unittest.TestCase): - def setUp(self): self.mock_cache = Mock() self.mock_create_factory = Mock() @@ -307,17 +313,19 @@ def setUp(self): def get_or_set(self, dependency_key_factory=None): from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns') + + cached = Cached(self.mock_cache, time=10, namespace="ns") cached.dependency = self.mock_dependency - return cached.get_or_set('key', self.mock_create_factory, - dependency_key_factory) + return cached.get_or_set( + "key", self.mock_create_factory, dependency_key_factory + ) def test_found(self): """ An item found in cache. """ - self.mock_cache.get.return_value = 'x' - assert 'x' == self.get_or_set() - self.mock_cache.get.assert_called_once_with('key', 'ns') + self.mock_cache.get.return_value = "x" + assert "x" == self.get_or_set() + self.mock_cache.get.assert_called_once_with("key", "ns") assert not self.mock_create_factory.called def test_create_none(self): @@ -327,7 +335,7 @@ def test_create_none(self): self.mock_create_factory.return_value = None assert not self.get_or_set() - self.mock_cache.get.assert_called_once_with('key', 'ns') + self.mock_cache.get.assert_called_once_with("key", "ns") self.mock_create_factory.assert_called_once_with() assert not self.mock_cache.add.called @@ -335,30 +343,29 @@ def test_no_dependency(self): """ There is specified `dependency_factory`. """ self.mock_cache.get.return_value = None - self.mock_create_factory.return_value = 'x' + self.mock_create_factory.return_value = "x" self.mock_cache.set.return_value = True - assert 'x' == self.get_or_set() - self.mock_cache.get.assert_called_once_with('key', 'ns') - self.mock_cache.set.assert_called_once_with('key', 'x', 10, 'ns') + assert "x" == self.get_or_set() + self.mock_cache.get.assert_called_once_with("key", "ns") + self.mock_cache.set.assert_called_once_with("key", "x", 10, "ns") def test_has_dependency(self): """ There is specified `dependency_factory`. """ self.mock_cache.get.return_value = None - self.mock_create_factory.return_value = 'x' + self.mock_create_factory.return_value = "x" self.mock_cache.set.return_value = True mock_dependency_key_factory = Mock() - mock_dependency_key_factory.return_value = 'master_key' + mock_dependency_key_factory.return_value = "master_key" - assert 'x' == self.get_or_set(mock_dependency_key_factory) - self.mock_cache.get.assert_called_once_with('key', 'ns') - self.mock_cache.set.assert_called_once_with('key', 'x', 10, 'ns') - self.mock_dependency.add.assert_called_once_with('master_key', 'key') + assert "x" == self.get_or_set(mock_dependency_key_factory) + self.mock_cache.get.assert_called_once_with("key", "ns") + self.mock_cache.set.assert_called_once_with("key", "x", 10, "ns") + self.mock_dependency.add.assert_called_once_with("master_key", "key") class WrapsGetOrSetTestCase(GetOrSetTestCase): - def test_has_dependency(self): pass @@ -367,26 +374,28 @@ def get_or_set(self, dependency_factory=None): def kb(f): def key(*args, **kwargs): - return 'key' + return "key" + return key - cached = Cached(self.mock_cache, kb, time=10, namespace='ns') + + cached = Cached(self.mock_cache, kb, time=10, namespace="ns") return cached.wraps_get_or_set(self.mock_create_factory)() class WrapsGetOrSetMakeKeyTestCase(WrapsGetOrSetTestCase): - def get_or_set(self, dependency_factory=None): from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns') - @cached.wraps_get_or_set(make_key=lambda: 'key') + cached = Cached(self.mock_cache, time=10, namespace="ns") + + @cached.wraps_get_or_set(make_key=lambda: "key") def create_factory(): return self.mock_create_factory() + return create_factory() class CachedCallTestCase(GetOrSetTestCase): - def setUp(self): self.mock_cache = Mock() self.mock_create_factory = Mock() @@ -396,19 +405,19 @@ def get_or_set(self, dependency_factory=None): def kb(f): def key(*args, **kwargs): - return 'key' + return "key" + return key - cached = Cached(self.mock_cache, kb, time=10, namespace='ns') + + cached = Cached(self.mock_cache, kb, time=10, namespace="ns") return cached(self.mock_create_factory)() def test_has_dependency(self): """ Not supported. """ - pass class OnePassCreateTestCase(unittest.TestCase): - def setUp(self): self.mock_cache = Mock() self.mock_create_factory = Mock() @@ -416,10 +425,12 @@ def setUp(self): def one_pass_create(self, dependency_key_factory=None): from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns', timeout=5) + + cached = Cached(self.mock_cache, time=10, namespace="ns", timeout=5) cached.dependency = self.mock_dependency - return cached.one_pass_create('key', self.mock_create_factory, - dependency_key_factory) + return cached.one_pass_create( + "key", self.mock_create_factory, dependency_key_factory + ) def test_create_none(self): """ One pass has been entered and create factory returns None. @@ -429,44 +440,44 @@ def test_create_none(self): assert not self.one_pass_create() self.mock_cache.add.assert_called_once_with( - 'one_pass:key', ANY, 5, 'ns') + "one_pass:key", ANY, 5, "ns" + ) self.mock_create_factory.assert_called_once_with() assert not self.mock_cache.set.called - self.mock_cache.delete.assert_called_once_with( - 'one_pass:key', 0, 'ns') + self.mock_cache.delete.assert_called_once_with("one_pass:key", 0, "ns") def test_no_dependency(self): """ Create factory returned value. """ self.mock_cache.add.return_value = True - self.mock_create_factory.return_value = 'x' - assert 'x' == self.one_pass_create() + self.mock_create_factory.return_value = "x" + assert "x" == self.one_pass_create() self.mock_cache.add.assert_called_once_with( - 'one_pass:key', ANY, 5, 'ns') + "one_pass:key", ANY, 5, "ns" + ) self.mock_create_factory.assert_called_once_with() - self.mock_cache.set.assert_called_once_with('key', 'x', 10, 'ns') - self.mock_cache.delete.assert_called_once_with( - 'one_pass:key', 0, 'ns') + self.mock_cache.set.assert_called_once_with("key", "x", 10, "ns") + self.mock_cache.delete.assert_called_once_with("one_pass:key", 0, "ns") def test_with_dependency(self): """ Create factory returned value. """ self.mock_cache.add.return_value = True - self.mock_create_factory.return_value = 'x' + self.mock_create_factory.return_value = "x" mock_dependency_key_factory = Mock() - mock_dependency_key_factory.return_value = 'master_key' - assert 'x' == self.one_pass_create(mock_dependency_key_factory) + mock_dependency_key_factory.return_value = "master_key" + assert "x" == self.one_pass_create(mock_dependency_key_factory) self.mock_cache.add.assert_called_once_with( - 'one_pass:key', ANY, 5, 'ns') + "one_pass:key", ANY, 5, "ns" + ) self.mock_create_factory.assert_called_once_with() - self.mock_cache.set.assert_called_once_with('key', 'x', 10, 'ns') - self.mock_cache.delete.assert_called_once_with( - 'one_pass:key', 0, 'ns') - self.mock_dependency.add.assert_called_once_with('master_key', 'key') + self.mock_cache.set.assert_called_once_with("key", "x", 10, "ns") + self.mock_cache.delete.assert_called_once_with("one_pass:key", 0, "ns") + self.mock_dependency.add.assert_called_once_with("master_key", "key") - @patch('wheezy.caching.patterns.OnePass') + @patch("wheezy.caching.patterns.OnePass") def test_wait_timedout(self, mock_cls_one_pass): """ Wait on one pass has timed out. """ @@ -477,7 +488,7 @@ def test_wait_timedout(self, mock_cls_one_pass): assert not self.one_pass_create() - @patch('wheezy.caching.patterns.OnePass') + @patch("wheezy.caching.patterns.OnePass") def test_wait_get(self, mock_cls_one_pass): """ Wait on one pass succeed, get value. """ @@ -485,67 +496,69 @@ def test_wait_get(self, mock_cls_one_pass): self.mock_cache.add.return_value = False mock_one_pass.acquired = False mock_one_pass.wait.return_value = True - self.mock_cache.get.return_value = 'x' + self.mock_cache.get.return_value = "x" - assert 'x' == self.one_pass_create() + assert "x" == self.one_pass_create() class GetOrCreateTestCase(unittest.TestCase): - def setUp(self): self.mock_cache = Mock() self.mock_create_factory = Mock() def get_or_create(self): from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns', timeout=5) - return cached.get_or_create('key', self.mock_create_factory) + + cached = Cached(self.mock_cache, time=10, namespace="ns", timeout=5) + return cached.get_or_create("key", self.mock_create_factory) def test_found(self): """ An item found in cache. """ - self.mock_cache.get.return_value = 'x' - assert 'x' == self.get_or_create() - self.mock_cache.get.assert_called_once_with('key', 'ns') + self.mock_cache.get.return_value = "x" + assert "x" == self.get_or_create() + self.mock_cache.get.assert_called_once_with("key", "ns") assert not self.mock_create_factory.called def test_not_found(self): """ Not found in cache. """ self.mock_cache.get.return_value = None - self.mock_create_factory.return_value = 'x' - assert 'x' == self.get_or_create() - self.mock_cache.get.assert_called_once_with('key', 'ns') + self.mock_create_factory.return_value = "x" + assert "x" == self.get_or_create() + self.mock_cache.get.assert_called_once_with("key", "ns") class WrapsGetOrCreateTestCase(GetOrCreateTestCase): - def get_or_create(self, dependency_factory=None): from wheezy.caching.patterns import Cached def kb(f): def key(*args, **kwargs): - return 'key' + return "key" + return key - cached = Cached(self.mock_cache, kb, time=10, namespace='ns', - timeout=5) + + cached = Cached( + self.mock_cache, kb, time=10, namespace="ns", timeout=5 + ) return cached.wraps_get_or_create(self.mock_create_factory)() class WrapsGetOrCreateMakeKeyTestCase(WrapsGetOrCreateTestCase): - def get_or_create(self, dependency_factory=None): from wheezy.caching.patterns import Cached - cached = Cached(self.mock_cache, time=10, namespace='ns') - @cached.wraps_get_or_create(make_key=lambda: 'key') + cached = Cached(self.mock_cache, time=10, namespace="ns") + + @cached.wraps_get_or_create(make_key=lambda: "key") def create_factory(): return self.mock_create_factory() + return create_factory() class GetOrSetMultiTestCase(unittest.TestCase): - def setUp(self): self.mock_cache = Mock() self.mock_create_factory = Mock() @@ -554,18 +567,20 @@ def get_or_set_multi(self): from wheezy.caching.patterns import Cached def mk(i): - return 'k%d' % i - cached = Cached(self.mock_cache, time=10, namespace='ns') - r = cached.get_or_set_multi( - mk, self.mock_create_factory, [1, 2]) - assert [(1, 'a'), (2, 'b')] == sorted(r.items()) + return "k%d" % i + + cached = Cached(self.mock_cache, time=10, namespace="ns") + r = cached.get_or_set_multi(mk, self.mock_create_factory, [1, 2]) + assert [(1, "a"), (2, "b")] == sorted(r.items()) def test_all_cache_hit(self): """ All items are taken from cache. """ + def get_multi(keys, namespace): - assert ['k1', 'k2'] == sorted(keys) - return {'k1': 'a', 'k2': 'b'} + assert ["k1", "k2"] == sorted(keys) + return {"k1": "a", "k2": "b"} + self.mock_cache.get_multi.side_effect = get_multi self.get_or_set_multi() assert not self.mock_create_factory.called @@ -573,10 +588,12 @@ def get_multi(keys, namespace): def test_all_cache_miss(self): """ All items are missed in cache. """ + def set_multi(keys, time, namespace): - assert [('k1', 'a'), ('k2', 'b')] == sorted(keys.items()) + assert [("k1", "a"), ("k2", "b")] == sorted(keys.items()) + self.mock_cache.get_multi.return_value = {} - self.mock_create_factory.return_value = {1: 'a', 2: 'b'} + self.mock_create_factory.return_value = {1: "a", 2: "b"} self.mock_cache.set_multi.side_effect = set_multi self.get_or_set_multi() self.mock_create_factory.assert_called_once_with([1, 2]) @@ -585,12 +602,13 @@ def set_multi(keys, time, namespace): def test_some_cache_miss(self): """ Some items are missed in cache. """ - self.mock_cache.get_multi.return_value = {'k2': 'b'} - self.mock_create_factory.return_value = {1: 'a'} + self.mock_cache.get_multi.return_value = {"k2": "b"} + self.mock_create_factory.return_value = {1: "a"} self.get_or_set_multi() self.mock_create_factory.assert_called_once_with([1]) self.mock_cache.set_multi.assert_called_once_with( - {'k1': 'a'}, 10, 'ns') + {"k1": "a"}, 10, "ns" + ) def test_some_cache_miss_create_factory_no_result(self): """ Some items are missed in cache and factory returns @@ -599,87 +617,94 @@ def test_some_cache_miss_create_factory_no_result(self): from wheezy.caching.patterns import Cached def mk(i): - return 'k%d' % i - cached = Cached(self.mock_cache, time=10, namespace='ns') - self.mock_cache.get_multi.return_value = {'k2': 'b'} + return "k%d" % i + + cached = Cached(self.mock_cache, time=10, namespace="ns") + self.mock_cache.get_multi.return_value = {"k2": "b"} self.mock_create_factory.return_value = {} - r = cached.get_or_set_multi( - mk, self.mock_create_factory, [1, 2]) - assert [(2, 'b')] == list(r.items()) + r = cached.get_or_set_multi(mk, self.mock_create_factory, [1, 2]) + assert [(2, "b")] == list(r.items()) self.mock_create_factory.assert_called_once_with([1]) assert not self.mock_cache.set_multi.called class WrapsGetOrSetMultiTestCase(GetOrSetMultiTestCase): - def get_or_set_multi(self): from wheezy.caching.patterns import Cached def mk(i): - return 'k%d' % i - cached = Cached(self.mock_cache, time=10, namespace='ns') + return "k%d" % i + + cached = Cached(self.mock_cache, time=10, namespace="ns") @cached.wraps_get_or_set_multi(make_key=mk) def create_factory(ids): return self.mock_create_factory(ids) + r = create_factory([1, 2]) - assert [(1, 'a'), (2, 'b')] == sorted(r.items()) + assert [(1, "a"), (2, "b")] == sorted(r.items()) class WrapsGetOrSetMultiCtxTestCase(GetOrSetMultiTestCase): - def get_or_set_multi(self): from wheezy.caching.patterns import Cached def mk(i): - return 'k%d' % i - cached = Cached(self.mock_cache, time=10, namespace='ns') + return "k%d" % i + + cached = Cached(self.mock_cache, time=10, namespace="ns") @cached.wraps_get_or_set_multi(make_key=mk) def create_factory(cls, ids): return self.mock_create_factory(ids) - r = create_factory('cls', [1, 2]) - assert [(1, 'a'), (2, 'b')] == sorted(r.items()) + r = create_factory("cls", [1, 2]) + assert [(1, "a"), (2, "b")] == sorted(r.items()) -class KeyBuilderTestCase(unittest.TestCase): +class KeyBuilderTestCase(unittest.TestCase): def setUp(self): from wheezy.caching.patterns import key_builder - self.mk = key_builder('prefix') + + self.mk = key_builder("prefix") def test_noargs(self): def items(): pass # pragma: nocover - assert 'prefix-items' == self.mk(items)() + + assert "prefix-items" == self.mk(items)() def test_args(self): def items(x, y): pass # pragma: nocover - assert "prefix-items:'None':None" == self.mk(items)('None', None) + + assert "prefix-items:'None':None" == self.mk(items)("None", None) def test_defaults(self): - def items(x, y=''): + def items(x, y=""): pass # pragma: nocover + assert "prefix-items:1:''" == self.mk(items)(1) - assert "prefix-items:1:'s'" == self.mk(items)(1, y='s') + assert "prefix-items:1:'s'" == self.mk(items)(1, y="s") assert "prefix-items:1:2" == self.mk(items)(1, y=2) def test_sepecial(self): def items(cls, y): pass # pragma: nocover - assert 'prefix-items:1' == self.mk(items)('cls', 1) - assert 'prefix-items:None' == self.mk(items)('cls', None) - assert "prefix-items:''" == self.mk(items)('cls', '') + + assert "prefix-items:1" == self.mk(items)("cls", 1) + assert "prefix-items:None" == self.mk(items)("cls", None) + assert "prefix-items:''" == self.mk(items)("cls", "") def test_object(self): class Spec(object): - def __init__(self, locale='en'): + def __init__(self, locale="en"): self.locale = locale def __repr__(self): - return '' % self.locale + return "" % self.locale def items(spec): pass # pragma: nocover - assert 'prefix-items:' == self.mk(items)(Spec()) + + assert "prefix-items:" == self.mk(items)(Spec()) diff --git a/src/wheezy/caching/tests/test_pylibmc.py b/src/wheezy/caching/tests/test_pylibmc.py index b113571..3472dd0 100644 --- a/src/wheezy/caching/tests/test_pylibmc.py +++ b/src/wheezy/caching/tests/test_pylibmc.py @@ -1,15 +1,12 @@ - """ Unit tests for ``wheezy.caching.pylibmc``. """ import os from unittest import TestCase -from wheezy.caching.comp import Queue -from wheezy.caching.comp import xrange +from wheezy.caching.comp import Queue, xrange from wheezy.caching.tests.test_cache import CacheTestMixin - try: from wheezy.caching.pylibmc import client_factory except ImportError: # pragma: nocover @@ -18,7 +15,6 @@ from wheezy.caching.pylibmc import MemcachedClient class EagerPool(object): - def __init__(self, create_factory, size): pool = Queue(size) for _ in xrange(size): @@ -28,12 +24,13 @@ def __init__(self, create_factory, size): self.get_back = pool.put client_pool = EagerPool( - lambda: client_factory([ - os.environ.get('MEMCACHED_HOST', '127.0.0.1') - ]), 1) + lambda: client_factory( + [os.environ.get("MEMCACHED_HOST", "127.0.0.1")] + ), + 1, + ) class PylibmcClientTestCase(TestCase, CacheTestMixin): - def setUp(self): self.client = MemcachedClient(client_pool) self.namespace = None @@ -42,7 +39,7 @@ def tearDown(self): self.client.flush_all() def test_delete_multi(self): - mapping = {'d1': 1, 'd2': 2} + mapping = {"d1": 1, "d2": 2} keys = list(mapping.keys()) assert not self.client.delete_multi(keys) self.setget_multi(mapping) diff --git a/src/wheezy/caching/utils.py b/src/wheezy/caching/utils.py index 71603af..adfed7d 100644 --- a/src/wheezy/caching/utils.py +++ b/src/wheezy/caching/utils.py @@ -1,4 +1,3 @@ - """ ``utils`` module. """ @@ -30,5 +29,6 @@ def total_seconds(delta): elif isinstance(delta, timedelta): return delta.seconds + delta.days * 86400 else: - raise TypeError('Expecting type datetime.timedelta ' - 'or int for seconds') + raise TypeError( + "Expecting type datetime.timedelta " "or int for seconds" + ) diff --git a/tox.ini b/tox.ini index 8557ecd..5a7853b 100644 --- a/tox.ini +++ b/tox.ini @@ -18,12 +18,19 @@ commands = basepython = python3 skip_install = True deps = + autoflake + black flake8 flake8-bugbear flake8-import-order flake8-mutable + isort pep8-naming commands = + autoflake --in-place --remove-unused-variables --remove-all-unused-imports \ + --recursive src/ setup.py + isort --profile black --combine-as --case-sensitive src/ setup.py + black -ql79 src/ setup.py flake8 doc src setup.py [testenv:docs]