Merge 8ef108c into 5d41a09

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.0
+current_version = 2.1.0
 commit = True
 tag = True
 message = 🔖 Bump version: {current_version} → {new_version}

.github/workflows/python.yml

@@ -15,7 +15,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        # python-version: ["3.11", "3.12"]
+        python-version: ["3.11"]
 
     steps:
     - uses: actions/checkout@v4
@@ -67,4 +68,5 @@ jobs:
       uses: coverallsapp/github-action@v2
       with:
         parallel-finished: true
-        carryforward: "run-3.10,run-3.11"
+        # carryforward: "run-3.11,run-3.12"
+        carryforward: "run-3.11"

CHANGELOG.md

@@ -1,5 +1,11 @@
 # ftmstore-fastapi
 
+## 2.1.0 (2023-10-19)
+
+- Drop support for Python 3.10 due to dependencies
+- Implement aggregation endpoints
+- Use store resolver
+
 ## 2.0.0 (2023-09-28)
 
 - Build on top of [ftmq](https://github.com/investigativedata/ftmq/)

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/investigativedata/ftm-docker:main
+FROM ghcr.io/investigativedata/ftm-docker:latest
 
 COPY ftmstore_fastapi /app/ftmstore_fastapi
 COPY setup.py /app/setup.py

VERSION

@@ -1 +1 @@
-2.0.0
+2.1.0

ftmstore_fastapi/__init__.py

@@ -1 +1 @@
-__version__ = "2.0.0"
+__version__ = "2.1.0"

ftmstore_fastapi/api.py

@@ -21,6 +21,7 @@
 log = get_logger(__name__)
 
 app = FastAPI(
+    debug=settings.DEBUG,
     title=settings.TITLE,
     contact=settings.CONTACT,
     description=settings.DESCRIPTION,

ftmstore_fastapi/query.py

@@ -1,13 +1,12 @@
 from typing import Annotated, Any
 
 from banal import clean_dict
-from fastapi import HTTPException
 from fastapi import Query as FastQuery
 from fastapi import Request
 from ftmq.aggregations import Aggregator
-from ftmq.enums import Schemata
 from ftmq.query import Query as _Query
-from pydantic import BaseModel, Field, validator
+from ftmq.types import Schemata
+from pydantic import BaseModel, Field
 
 from ftmstore_fastapi import settings
 from ftmstore_fastapi.store import Datasets
@@ -25,6 +24,8 @@ class AggregationParams(BaseModel):
     aggMin: list[str] | None = []
     aggMax: list[str] | None = []
     aggAvg: list[str] | None = []
+    aggCount: list[str] | None = []
+    aggGroups: list[str] | None = []
 
 
 class QueryParams(BaseModel):
@@ -48,12 +49,6 @@ class QueryParams(BaseModel):
     class Config:
         allow_population_by_field_name = True
 
-    @validator("schema_")
-    def validate_schema(cls, value: str | None) -> bool:
-        if value is not None and value not in Schemata:
-            raise HTTPException(400, detail=[f"Invalid ftm schema: `{value}`"])
-        return value
-
     def to_where_lookup_dict(self) -> dict[str, Any]:
         return {k: v for k, v in self.dict().items() if v and k not in META_FIELDS}
 
@@ -64,6 +59,8 @@ def to_where_lookup_dict(self) -> dict[str, Any]:
     | set(QueryParams.__fields__)  # noqa: W503
 )
 
+LISTISH_PARAMS = ["dataset", *AggregationParams.__fields__.keys()]
+
 
 class ViewQueryParams(QueryParams):
     class Config:
@@ -79,7 +76,11 @@ def from_request(
         cls, request: Request, authenticated: bool | None = False
     ) -> "ViewQueryParams":
         params = dict(request.query_params)
-        params["dataset"] = request.query_params.getlist("dataset")
+        # listish params
+        for p in LISTISH_PARAMS:
+            listish = request.query_params.getlist(p)
+            if listish:
+                params[p] = listish
         params = cls(**params)
         if not authenticated and params.limit > settings.DEFAULT_LIMIT:
             params.limit = settings.DEFAULT_LIMIT
@@ -101,14 +102,14 @@ def from_params(cls: "Query", params: ViewQueryParams) -> "Query":
         q = cls()[(params.page - 1) * params.limit : params.page * params.limit]
         if params.dataset:
             q = q.where(dataset__in=params.dataset)
+        if params.schema_:
+            q = q.where(schema=params.schema_)
         if params.order_by:
             ascending = True
             if params.order_by.startswith("-"):
                 ascending = False
                 params.order_by = params.order_by.lstrip("-")
             q = q.order_by(params.order_by, ascending=ascending)
-        if params.schema_:
-            q = q.where(schema=params.schema_)
         if params.reverse:
             q = q.where(reverse=params.reverse)
         q = q.where(**params.to_where_lookup_dict())

ftmstore_fastapi/serialize.py

@@ -9,7 +9,7 @@
 
 from banal import clean_dict
 from fastapi import Request
-from followthemoney.model import registry
+from followthemoney.types import registry
 from ftmq.aggregations import AggregatorResult
 from ftmq.model import Catalog, Coverage, Dataset
 from ftmq.types import CE, CEGenerator

ftmstore_fastapi/settings.py

@@ -1,5 +1,4 @@
 import os
-from pathlib import Path
 
 from banal import as_bool
 from nomenklatura.settings import DB_URL
@@ -9,8 +8,7 @@
 VERSION = __version__
 
 CATALOG = os.environ.get("CATALOG")
-if CATALOG is not None:
-    CATALOG = Path(CATALOG)
+RESOLVER = os.environ.get("RESOLVER", os.environ.get("RESOLVER_PATH"))
 
 FTM_STORE_URI = os.environ.get("FTM_STORE_URI", DB_URL)
 

ftmstore_fastapi/store.py

@@ -1,15 +1,16 @@
-from functools import cache
+from functools import cache, lru_cache
 from typing import TYPE_CHECKING, Literal
 
 from fastapi import HTTPException
+from ftmq.dedupe import get_resolver
 from ftmq.model import Catalog, Dataset
 from ftmq.query import Q
 from ftmq.store import Store
 from ftmq.store import get_store as _get_store
 from ftmq.types import CE, CEGenerator
 
 from ftmstore_fastapi.logging import get_logger
-from ftmstore_fastapi.settings import CATALOG, FTM_STORE_URI
+from ftmstore_fastapi.settings import CATALOG, FTM_STORE_URI, RESOLVER
 from ftmstore_fastapi.util import get_dehydrated_proxy, get_featured_proxy
 
 if TYPE_CHECKING:
@@ -36,29 +37,44 @@ def get_dataset(name: str, catalog: Catalog | None = None) -> Dataset:
 
 
 @cache
-def get_store(dataset: str | None = None, catalog_uri: str | None = None) -> Store:
-    catalog = get_catalog(catalog_uri)
+def get_store(
+    dataset: str | None = None,
+    catalog_uri: str | None = None,
+    resolver_uri: str | None = None,
+) -> Store:
+    catalog = get_catalog(catalog_uri or CATALOG)
+    resolver = get_resolver(resolver_uri or RESOLVER)
     if dataset is not None:
         dataset = get_dataset(dataset, catalog)
-        return _get_store(catalog=catalog, dataset=dataset, uri=FTM_STORE_URI)
-    return _get_store(catalog=catalog, uri=FTM_STORE_URI)
+        store = _get_store(
+            catalog=catalog, dataset=dataset, uri=FTM_STORE_URI, resolver=resolver
+        )
+    else:
+        store = _get_store(catalog=catalog, uri=FTM_STORE_URI, resolver=resolver)
+    return store
 
 
 class View:
     def __init__(
-        self, dataset: str | None = None, catalog_uri: str | None = None
+        self,
+        dataset: str | None = None,
+        catalog_uri: str | None = None,
+        resolver_uri: str | None = None,
     ) -> None:
-        store = get_store(dataset, catalog_uri)
+        self.store = get_store(dataset, catalog_uri, resolver_uri)
         self.dataset = dataset
-        self.query = store.query()
-        self.view = store.default_view()
+        self.query = self.store.query()
+        self.view = self.store.default_view()
 
         self.coverage = self.query.coverage
         self.aggregations = self.query.aggregations
         self.get_adjacents = self.query.get_adjacents
 
     def get_entity(self, entity_id: str, params: "RetrieveParams") -> CE | None:
-        proxy = self.view.get_entity(entity_id)
+        canonical = self.store.resolver.get_canonical(entity_id)
+        proxy = get_cached_entity(self.view, canonical)
+        if proxy is None:
+            raise HTTPException(404, detail=[f"Entity `{entity_id}` not found."])
         if params.dehydrate:
             return get_dehydrated_proxy(proxy)
         if params.featured:
@@ -75,8 +91,17 @@ def get_entities(self, query: Q, params: "RetrieveParams") -> CEGenerator:
 
 
 @cache
-def get_view(dataset: str | None = None, catalog_uri: str | None = None) -> View:
-    return View(dataset, catalog_uri)
+def get_view(
+    dataset: str | None = None,
+    catalog_uri: str | None = None,
+    resolver_uri: str | None = None,
+) -> View:
+    return View(dataset, catalog_uri, resolver_uri)
+
+
+@lru_cache(10_000)
+def get_cached_entity(view: View, entity_id: str) -> CE:
+    return view.get_entity(entity_id)
 
 
 # cache at boot time