Merge branch 'aioredis' into main

HISTORY.md

@@ -1,8 +1,9 @@
 # History
 
 ## 0.2.0 (TBD)
-* Update SQLite backend to use `aiosqlite` for async cache operations
-* Update MongoDB backend to use `motor` for async cache operations
+* Refactor SQLite backend to use `aiosqlite` for async cache operations
+* Refactor MongoDB backend to use `motor` for async cache operations
+* Refactor Redis backend to use `aiosqlite` for async cache operations
 
 ## 0.1.0 (2020-11-14)
 * Initial PyPI release

README.md

@@ -18,9 +18,8 @@ for the aiohttp web server. This package is, as you might guess, specifically fo
 ## Development Status
 **This is an early work in progress!**
 
-The current state is a working drop-in replacement (or mixin) for `aiohttp.ClientSession`, with a
-fully async SQLite backend.
-Additional backends are provided, but cache operations are still synchronous, and have had minimal testing.
+The current state is a working drop-in replacement (or mixin) for `aiohttp.ClientSession`, with
+multiple asynchronous cache backends.
 
 Breaking changes should be expected until a `1.0` release.
 
@@ -68,19 +67,21 @@ class CustomSession(CacheMixin, CustomMixin, ClientSession):
 ```
 
 ## Cache Backends
-Several backends are available. If one isn't specified, a simple in-memory cache will be used.
+Several backends are available. If one isn't specified, a non-persistent in-memory cache will be used.
 
 * `SQLiteBackend`: Uses a [SQLite](https://www.sqlite.org) database
   (requires [aiosqlite](https://github.com/omnilib/aiosqlite))
-* `DynamoDBBackend`: Uses a [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) database
-  (requires [boto3](https://github.com/boto/boto3))
 * `RedisBackend`: Uses a [Redis](https://redis.io/) cache
   (requires [redis-py](https://github.com/andymccurdy/redis-py))
 * `MongoDBBackend`: Uses a [MongoDB](https://www.mongodb.com/) database
   (requires [motor](https://motor.readthedocs.io))
-    * `GridFSBackend`: Uses a [MongoDB GridFS](https://docs.mongodb.com/manual/core/gridfs/) database,
-      which enables storage of documents greater than 16MB
-      (requires [pymongo](https://pymongo.readthedocs.io/en/stable/))
+
+**Incomplete:**
+* `DynamoDBBackend`: Uses a [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) database
+  (requires [boto3](https://github.com/boto/boto3))
+* `GridFSBackend`: Uses a [MongoDB GridFS](https://docs.mongodb.com/manual/core/gridfs/) database,
+  which enables storage of documents greater than 16MB
+  (requires [pymongo](https://pymongo.readthedocs.io/en/stable/))
 
 You can also provide your own backend by subclassing `aiohttp_client_cache.backends.BaseCache`.
 

aiohttp_client_cache/backends/base.py

@@ -52,7 +52,7 @@ class CacheBackend:
 
     def __init__(
         self,
-        cache_name: str = 'http-cache',
+        cache_name: str = 'aiohttp-cache',
         expire_after: Union[int, timedelta] = None,
         allowed_codes: tuple = (200,),
         allowed_methods: tuple = ('GET', 'HEAD'),

aiohttp_client_cache/backends/dynamodb.py

@@ -20,10 +20,12 @@ class DynamoDBBackend(CacheBackend):
     """
 
     @extend_signature(CacheBackend.__init__)
-    def __init__(self, cache_name: str = 'http-cache', **kwargs):
+    def __init__(self, cache_name: str = 'aiohttp-cache', **kwargs):
         super().__init__(cache_name=cache_name, **kwargs)
         self.responses = DynamoDbCache(cache_name, 'responses', **kwargs)
-        self.redirects = DynamoDbCache(cache_name, 'urls', connection=self.responses.connection)
+        self.redirects = DynamoDbCache(
+            cache_name, 'redirects', connection=self.responses.connection
+        )
 
 
 # TODO: Incomplete/untested

aiohttp_client_cache/backends/gridfs.py

@@ -20,10 +20,10 @@ class GridFSBackend(CacheBackend):
     """
 
     @extend_signature(CacheBackend.__init__)
-    def __init__(self, cache_name: str = 'http-cache', connection: MongoClient = None, **kwargs):
+    def __init__(self, cache_name: str = 'aiohttp-cache', connection: MongoClient = None, **kwargs):
         super().__init__(cache_name=cache_name, **kwargs)
         self.responses = GridFSCache(cache_name, connection)
-        self.keys_map = MongoDBCache(cache_name, 'http_redirects', self.responses.connection)
+        self.keys_map = MongoDBCache(cache_name, 'redirects', self.responses.connection)
 
 
 # TODO: Incomplete/untested

aiohttp_client_cache/backends/mongo.py

@@ -18,11 +18,11 @@ class MongoDBBackend(CacheBackend):
 
     @extend_signature(CacheBackend.__init__)
     def __init__(
-        self, cache_name: str = 'http-cache', connection: AsyncIOMotorClient = None, **kwargs
+        self, cache_name: str = 'aiohttp-cache', connection: AsyncIOMotorClient = None, **kwargs
     ):
         super().__init__(cache_name=cache_name, **kwargs)
         self.responses = MongoDBPickleCache(cache_name, 'responses', connection)
-        self.keys_map = MongoDBCache(cache_name, 'urls', self.responses.connection)
+        self.keys_map = MongoDBCache(cache_name, 'redirects', self.responses.connection)
 
 
 class MongoDBCache(BaseCache):

aiohttp_client_cache/backends/redis.py

@@ -1,76 +1,95 @@
 import pickle
 from typing import Iterable
 
-from redis import Redis, StrictRedis
+from aioredis import Redis, create_redis_pool
 
 from aiohttp_client_cache.backends import BaseCache, CacheBackend, ResponseOrKey
 from aiohttp_client_cache.forge_utils import extend_signature
 
+DEFAULT_ADDRESS = 'redis://localhost'
 
-class RedisBackend(CacheBackend):
-    """Redis cache backend.
 
-    See :py:class:`.CacheBackend` for args.
-    """
+class RedisBackend(CacheBackend):
+    """Redis cache backend"""
 
     @extend_signature(CacheBackend.__init__)
-    def __init__(self, cache_name: str = 'http-cache', **kwargs):
+    def __init__(self, cache_name: str = 'aiohttp-cache', address: str = DEFAULT_ADDRESS, **kwargs):
         super().__init__(cache_name=cache_name, **kwargs)
-        self.responses = RedisCache(cache_name, 'responses', **kwargs)
-        self.redirects = RedisCache(cache_name, 'urls', connection=self.responses.connection)
+        self.responses = RedisCache(cache_name, 'responses', address=address, **kwargs)
+        self.redirects = RedisCache(cache_name, 'redirects', address=address, **kwargs)
 
 
-# TODO: Incomplete/untested
-# TODO: Original implementation pickled keys as well as values. Is there a reason keys need to be pickled?
-# TODO: Fully async implementation. Current implementation with redis-py uses blocking operations.
-#   Methods are currently defined as async only for compatibility with BaseCache API.
 class RedisCache(BaseCache):
     """An async-compatible interface for caching objects in Redis.
-    The actual key name on the redis server will be ``namespace:collection_name``.
-    In order to deal with how redis stores data/keys, everything must be pickled.
 
     Args:
         namespace: namespace to use
         collection_name: name of the hash map stored in redis
         connection: An existing connection object to reuse instead of creating a new one
+        address: Address of Redis server
         kwargs: Additional keyword arguments for :py:class:`redis.Redis`
 
+    Note: The hash key name on the redis server will be ``namespace:collection_name``.
     """
 
-    def __init__(self, namespace: str, collection_name: str, connection: Redis = None, **kwargs):
-        self.connection = connection or StrictRedis(**kwargs)
-        self._self_key = ':'.join([namespace, collection_name])
+    def __init__(
+        self,
+        namespace: str,
+        collection_name: str,
+        address: str = None,
+        connection: Redis = None,
+        **kwargs,
+    ):
+        self.address = address
+        self._connection = connection
+        self.connection_kwargs = kwargs
+        self.hash_key = f'{namespace}:{collection_name}'
 
     @staticmethod
     def _unpickle_result(result):
         return pickle.loads(bytes(result)) if result else None
 
+    async def get_connection(self):
+        """Lazy-initialize redis connection"""
+        if not self._connection:
+            self._connection = await create_redis_pool(self.address, **self.connection_kwargs)
+        return self._connection
+
     async def clear(self):
-        self.connection.delete(self._self_key)
+        connection = await self.get_connection()
+        keys = await self.keys()
+        if keys:
+            await connection.hdel(self.hash_key, *keys)
 
     async def contains(self, key: str) -> bool:
-        return bool(self.connection.exists(key))
+        connection = await self.get_connection()
+        return await connection.hexists(self.hash_key, key)
 
     async def delete(self, key: str):
-        self.connection.hdel(self._self_key, pickle.dumps(key, protocol=-1))
+        connection = await self.get_connection()
+        await connection.hdel(self.hash_key, key)
 
     async def keys(self) -> Iterable[str]:
-        return [self._unpickle_result(r) for r in self.connection.hkeys(self._self_key)]
+        connection = await self.get_connection()
+        return [k.decode() for k in await connection.hkeys(self.hash_key)]
 
     async def read(self, key: str) -> ResponseOrKey:
-        result = self.connection.hget(self._self_key, pickle.dumps(key, protocol=-1))
+        connection = await self.get_connection()
+        result = await connection.hget(self.hash_key, key)
         return self._unpickle_result(result)
 
     async def size(self) -> int:
-        return self.connection.hlen(self._self_key)
+        connection = await self.get_connection()
+        return await connection.hlen(self.hash_key)
 
-    # TODO
     async def values(self) -> Iterable[ResponseOrKey]:
-        raise NotImplementedError
+        connection = await self.get_connection()
+        return [self._unpickle_result(v) for v in await connection.hvals(self.hash_key)]
 
     async def write(self, key: str, item: ResponseOrKey):
-        self.connection.hset(
-            self._self_key,
-            pickle.dumps(key, protocol=-1),
+        connection = await self.get_connection()
+        await connection.hset(
+            self.hash_key,
+            key,
             pickle.dumps(item, protocol=-1),
         )

aiohttp_client_cache/backends/sqlite.py

@@ -25,13 +25,13 @@ class SQLiteBackend(CacheBackend):
     """
 
     @extend_signature(CacheBackend.__init__)
-    def __init__(self, cache_name: str = 'http-cache', **kwargs):
+    def __init__(self, cache_name: str = 'aiohttp-cache', **kwargs):
         super().__init__(cache_name=cache_name, **kwargs)
         path, ext = splitext(cache_name)
         cache_path = f'{path}.{ext or "sqlite"}'
 
-        self.redirects = SQLiteCache(cache_path, 'urls')
         self.responses = SQLitePickleCache(cache_path, 'responses')
+        self.redirects = SQLiteCache(cache_path, 'redirects')
 
 
 class SQLiteCache(BaseCache):

examples/preache.py → examples/precache.py

@@ -25,7 +25,7 @@
 from contextlib import contextmanager
 from os.path import getsize
 
-from aiohttp_client_cache import CachedSession
+from aiohttp_client_cache import CachedSession, SQLiteBackend
 
 CACHE_NAME = 'precache'
 DEFAULT_URL = 'https://www.nytimes.com'
@@ -34,7 +34,7 @@
 
 async def precache_page_links(parent_url):
     """Fetch and cache the content of a given web page and all links found on that page"""
-    async with CachedSession(backend='sqlite', cache_name='precache') as session:
+    async with CachedSession(cache=SQLiteBackend()) as session:
         urls = await get_page_links(session, parent_url)
 
         tasks = [asyncio.create_task(cache_url(session, url)) for url in urls]

setup.py

@@ -8,7 +8,7 @@
     # Packages used for CI jobs
     'build': ['coveralls', 'twine', 'wheel'],
     # Packages for all supported backends
-    'backends': ['aiosqlite', 'boto3', 'motor', 'redis'],
+    'backends': ['aiosqlite', 'boto3', 'motor', 'aioredis'],
     # Packages used for documentation builds
     'docs': [
         'm2r2',

test/integration/backends/test_backend_mongo.py → test/integration/test_mongo_backend.py

@@ -23,20 +23,14 @@ def is_db_running():
     pytest.mark.skipif(not is_db_running(), reason='MongoDB server required for integration tests'),
 ]
 
-db_name = 'aiohttp-cache'
-collection_name = 'responses'
-
 
 @pytest.fixture(autouse=True, scope='function')
-async def cache_client(event_loop):
-    # We need to recreate the Motor client for every test method,
-    # else it will be using a different event loop than pytest.
-    connection = AsyncIOMotorClient()
-    cache_client = MongoDBCache(db_name, collection_name, connection)
-
-    await connection.drop_database(db_name)
+async def cache_client():
+    """Fixture that creates a new db client for each test function"""
+    cache_client = MongoDBCache('aiohttp-cache', 'responses')
+    await cache_client.clear()
     yield cache_client
-    await connection.drop_database(db_name)
+    await cache_client.clear()
 
 
 async def test_clear(cache_client):

test/integration/test_redis_backend.py

@@ -0,0 +1,84 @@
+import asyncio
+import pytest
+from datetime import datetime
+
+from aioredis import create_redis_pool
+
+from aiohttp_client_cache.backends.redis import DEFAULT_ADDRESS, RedisBackend, RedisCache
+
+
+def is_db_running():
+    """Test if a Redis server is running locally on the default port"""
+
+    async def get_db_info():
+        client = await create_redis_pool('redis://localhost')
+        await client.info()
+
+    try:
+        asyncio.run(get_db_info())
+        return True
+    except OSError:
+        return False
+
+
+pytestmark = [
+    pytest.mark.asyncio,
+    pytest.mark.skipif(not is_db_running(), reason='Redis server required for integration tests'),
+]
+
+test_data = {'key_1': 'item_1', 'key_2': datetime.now(), 'key_3': 3.141592654}
+
+
+@pytest.fixture(autouse=True, scope='function')
+async def cache_client():
+    """Fixture that creates a new db client for each test function"""
+    cache_client = RedisCache('aiohttp-cache', 'responses', 'redis://localhost')
+    await cache_client.clear()
+    yield cache_client
+    await cache_client.clear()
+
+
+def test_redis_backend():
+    backend = RedisBackend()
+    assert backend.responses.address == DEFAULT_ADDRESS
+    assert backend.responses.hash_key == 'aiohttp-cache:responses'
+    assert backend.redirects.hash_key == 'aiohttp-cache:redirects'
+
+
+async def test_write_read(cache_client):
+    # Test write() and contains()
+    for k, v in test_data.items():
+        await cache_client.write(k, v)
+        assert await cache_client.contains(k) is True
+
+    # Test read()
+    for k, v in test_data.items():
+        assert await cache_client.read(k) == v
+
+
+async def test_delete(cache_client):
+    for k, v in test_data.items():
+        await cache_client.write(k, v)
+
+    for k in test_data.keys():
+        await cache_client.delete(k)
+        assert await cache_client.contains(k) is False
+
+
+async def test_keys_values_size(cache_client):
+    for k, v in test_data.items():
+        await cache_client.write(k, v)
+
+    assert await cache_client.size() == len(test_data)
+    assert await cache_client.keys() == list(test_data.keys())
+    assert await cache_client.values() == list(test_data.values())
+
+
+async def test_clear(cache_client):
+    for k, v in test_data.items():
+        await cache_client.write(k, v)
+
+    await cache_client.clear()
+    assert await cache_client.size() == 0
+    assert await cache_client.keys() == []
+    assert await cache_client.values() == []