Merge 4d3fea7 into 7aa212b

app/api/v1/endpoints/domains.py

@@ -3,6 +3,8 @@
 
 import aioredis
 from fastapi import APIRouter, Depends
+from fastapi_cache.coder import PickleCoder
+from fastapi_cache.decorator import cache
 
 from app import schemas
 from app.core.dependencies import get_redis
@@ -11,12 +13,8 @@
 router = APIRouter()
 
 
-@router.get(
-    "/",
-    summary="Get the latest suspicious domains",
-    response_model=List[schemas.Domain],
-)
-async def get_domains(redis: aioredis.Redis = Depends(get_redis)):
+@cache(coder=PickleCoder, expire=60 * 5)
+async def _get_domains(redis: aioredis.Redis) -> List[schemas.Domain]:
     keys = await redis.keys(f"{KEY_PREFIX}*")
     if len(keys) == 0:
         return []
@@ -25,3 +23,12 @@ async def get_domains(redis: aioredis.Redis = Depends(get_redis)):
 
     dicts: List[dict] = [json.loads(value) for value in values]
     return [schemas.Domain.parse_obj(d) for d in dicts]
+
+
+@router.get(
+    "/",
+    summary="Get the latest suspicious domains",
+    response_model=List[schemas.Domain],
+)
+async def get_domains(redis: aioredis.Redis = Depends(get_redis)):
+    return await _get_domains(redis)

app/cache/backend.py

@@ -0,0 +1,71 @@
+import time
+from asyncio import Lock
+from dataclasses import dataclass
+from typing import Dict, Optional, Tuple
+
+from fastapi_cache.backends import Backend
+
+
+@dataclass
+class Value:
+    data: str
+    ttl_ts: Optional[int]
+
+
+class InMemoryBackend(Backend):
+    _store: Dict[str, Value] = {}
+    _lock = Lock()
+
+    @property
+    def _now(self) -> int:
+        return int(time.time())
+
+    def _get(self, key: str) -> Optional[Value]:
+        v = self._store.get(key)
+        if v:
+            if v.ttl_ts is not None and v.ttl_ts < self._now:
+                del self._store[key]
+            else:
+                return v
+
+        return None
+
+    async def get_with_ttl(self, key: str) -> Tuple[int, Optional[str]]:
+        async with self._lock:
+            v = self._get(key)
+            if v:
+                if v.ttl_ts is not None:
+                    return v.ttl_ts - self._now, v.data
+                else:
+                    return 0, v.data
+
+            return 0, None
+
+    async def get(self, key: str) -> Optional[str]:
+        async with self._lock:
+            v = self._get(key)
+            if v:
+                return v.data
+
+            return None
+
+    async def set(self, key: str, value: str, expire: Optional[int] = None) -> None:
+        async with self._lock:
+            if expire is not None:
+                self._store[key] = Value(value, self._now + expire)
+            else:
+                self._store[key] = Value(value, self._now)
+
+    async def clear(self, namespace: str = None, key: str = None) -> int:
+        count = 0
+        if namespace:
+            keys = list(self._store.keys())
+            for key in keys:
+                if key.startswith(namespace):
+                    del self._store[key]
+                    count += 1
+        elif key:
+            del self._store[key]
+            count += 1
+
+        return count

app/core/events.py

@@ -1,13 +1,30 @@
-from typing import Any, Callable, Coroutine
+from typing import Any, Callable, Coroutine, Union
 
+import aioredis
 from fastapi import FastAPI
+from fastapi_cache import FastAPICache
+from fastapi_cache.backends.redis import RedisBackend
+from loguru import logger
+
+from app.cache.backend import InMemoryBackend
+from app.core import settings
 
 
 def create_start_app_handler(
     app_: FastAPI,
 ) -> Callable[[], Coroutine[Any, Any, None]]:
     async def start_app() -> None:
-        pass
+        # initialize FastAPI cache
+        backend: Union[InMemoryBackend, RedisBackend] = InMemoryBackend()
+        if settings.REDIS_URL != "" and settings.TESTING is False:
+            try:
+                redis = await aioredis.create_redis_pool(str(settings.REDIS_URL))
+                backend = RedisBackend(redis)
+            except (ConnectionRefusedError, OSError) as e:
+                logger.error("Failed to connect to Redis")
+                logger.exception(e)
+
+        FastAPICache.init(backend, prefix="fastapi-cache")
 
     return start_app
 

pyproject.toml

@@ -11,6 +11,7 @@ aiofiles = "^0.7.00"
 arq = "^0.22"
 dataclasses-json = "^0.5.5"
 fastapi = "^0.68.1"
+fastapi-cache2 = {version = "0.1.3.4", extras = ["redis"]}
 gunicorn = "^20.1.0"
 httpx = "^0.19.0"
 loguru = "^0.5.3"

requirements.txt

@@ -1,46 +1,48 @@
 aiofiles==0.7.0; python_version >= "3.6" and python_version < "4.0"
-aioredis==1.3.1; python_version >= "3.6"
-anyio==3.3.0; python_full_version >= "3.6.2" and python_version >= "3.6"
+aioredis==1.3.1; python_version >= "3.7" and python_version < "4.0"
+anyio==3.3.1; python_full_version >= "3.6.2" and python_version >= "3.6"
 arq==0.22; python_version >= "3.6"
-asgiref==3.4.1; python_version >= "3.6"
+asgiref==3.4.1; python_version >= "3.7" and python_version < "4.0"
 async-timeout==3.0.1; python_full_version >= "3.5.3" and python_version >= "3.6"
 certifi==2021.5.30; python_version >= "3.6"
 cffi==1.14.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 charset-normalizer==2.0.4; python_full_version >= "3.5.0" and python_version >= "3.6"
-click==8.0.1; python_version >= "3.6"
-colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and platform_system == "Windows" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" and platform_system == "Windows"
+click==8.0.1; python_version >= "3.7" and python_version < "4.0"
+colorama==0.4.4; python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" and python_version < "4.0" and platform_system == "Windows" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0" and python_version < "4.0" and platform_system == "Windows"
 cryptography==3.4.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 dataclasses-json==0.5.5; python_version >= "3.6"
+fastapi-cache2==0.1.3.4; python_version >= "3.7" and python_version < "4.0"
 fastapi==0.68.1; python_version >= "3.6"
 gunicorn==20.1.0; python_version >= "3.5"
-h11==0.12.0; python_version >= "3.6"
+h11==0.12.0; python_version >= "3.7" and python_version < "4.0"
 hiredis==2.0.0; python_version >= "3.6"
 httpcore==0.13.6; python_version >= "3.6"
-httptools==0.2.0
+httptools==0.2.0; python_version >= "3.7" and python_version < "4.0"
 httpx==0.19.0; python_version >= "3.6"
 idna==3.2; python_version >= "3.6" and python_full_version >= "3.6.2"
 loguru==0.5.3; python_version >= "3.5"
 marshmallow-enum==1.5.1; python_version >= "3.6"
 marshmallow==3.13.0; python_version >= "3.6"
 mypy-extensions==0.4.3; python_version >= "3.6"
 pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pydantic==1.8.2; python_full_version >= "3.6.1" and python_version >= "3.6"
+pydantic==1.8.2; python_full_version >= "3.6.1" and python_version >= "3.7" and python_version < "4.0"
 pyhumps==3.0.2
 pyopenssl==20.0.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
-python-dotenv==0.19.0; python_version >= "3.5"
+python-dateutil==2.8.2; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.3.0"
+python-dotenv==0.19.0; python_version >= "3.7" and python_version < "4.0"
 python-levenshtein==0.12.2
-pyyaml==5.4.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
+pyyaml==5.4.1; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.0"
 rfc3986==1.5.0; python_version >= "3.6"
-six==1.16.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
+six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.5.0"
 sniffio==1.2.0; python_full_version >= "3.6.2" and python_version >= "3.6"
-starlette==0.14.2; python_version >= "3.6"
+starlette==0.14.2; python_version >= "3.7" and python_version < "4.0"
 tld==0.12.6; python_version >= "2.7" and python_version < "4"
 typing-extensions==3.10.0.2; python_full_version >= "3.6.1" and python_version >= "3.6"
 typing-inspect==0.7.1; python_version >= "3.6"
 ujson==4.1.0; python_version >= "3.6"
 uvicorn==0.15.0
-uvloop==0.16.0; sys_platform != "win32" and sys_platform != "cygwin" and platform_python_implementation != "PyPy" and python_version >= "3.7"
-watchgod==0.7; python_version >= "3.5"
+uvloop==0.16.0; sys_platform != "win32" and sys_platform != "cygwin" and platform_python_implementation != "PyPy" and python_version >= "3.7" and python_version < "4.0"
+watchgod==0.7; python_version >= "3.7" and python_version < "4.0"
 websocket-client==1.2.1; python_version >= "3.6"
-websockets==9.1; python_full_version >= "3.6.1"
+websockets==10.0; python_version >= "3.7" and python_version < "4.0"
 win32-setctime==1.0.3; sys_platform == "win32" and python_version >= "3.5"

tests/api/v1/endpoints/test_domains.py

@@ -1,8 +1,6 @@
-import httpx
-import pytest
+from fastapi.testclient import TestClient
 
 
-@pytest.mark.asyncio
-async def test_domains(client: httpx.AsyncClient):
-    res = await client.get("/api/v1/domains/")
+def test_domains(client: TestClient):
+    res = client.get("/api/v1/domains/")
     assert res.status_code == 200

tests/conftest.py

@@ -1,6 +1,6 @@
 import fakeredis.aioredis
-import httpx
 import pytest
+from fastapi.testclient import TestClient
 
 from app.core.dependencies import get_redis
 from app.main import create_app
@@ -11,10 +11,13 @@ async def override_get_redis():
 
 
 @pytest.fixture
-async def client() -> httpx.AsyncClient:
+def client() -> TestClient:
     app = create_app()
 
     app.dependency_overrides[get_redis] = override_get_redis
 
-    async with httpx.AsyncClient(app=app, base_url="http://testserver") as client:
-        yield client
+    with TestClient(
+        app=app,
+        base_url="http://testserver",
+    ) as c:
+        yield c

tests/views/test_index.py

@@ -1,18 +1,15 @@
-import httpx
-import pytest
+from fastapi.testclient import TestClient
 
 
-@pytest.mark.asyncio
-async def test_index(client: httpx.AsyncClient):
-    res = await client.get("/", allow_redirects=False)
+def test_index(client: TestClient):
+    res = client.get("/", allow_redirects=False)
 
     assert res.status_code == 302
     assert res.headers.get("location") == "/redoc"
 
 
-@pytest.mark.asyncio
-async def test_feed(client: httpx.AsyncClient):
-    res = await client.get("/feed", allow_redirects=False)
+def test_feed(client: TestClient):
+    res = client.get("/feed", allow_redirects=False)
 
     assert res.status_code == 302
     assert res.headers.get("location") == "/api/v1/domains/"