Log incoming HTTP requests (analytics) and add domain vision statemen…

…t (docs) (#316)

* add light api analytics

* add analytics middleware

* docs: domain vision statement draft

.gitignore

@@ -158,4 +158,6 @@ cordra-2.3.0/
 repl.ipynb
 *.crt
 
-tests/nmdcdb/
+tests/nmdcdb/
+
+neon_cache.sqlite

docs/explanation/domain-vision-statement.md

@@ -0,0 +1,40 @@
+# Domain Vision Statement
+
+The model will represent the status and nature of studies involving biosamples 
+in such a way that automated routing for computational analysis can be supported. 
+
+Biosamples are acquired from an environment;
+they are prepared for analysis using protocols;
+they are subject to analysis by instruments that output raw data;
+and they can be associated thematically with one or more studies.
+
+A study, conducted by a research team to answer research questions, collects
+(a) a set of biosamples, associated instrument-obtained ("raw") data,
+and/or associated computation-obtained ("derived") data; and
+(b) performs additional analysis activities in order to answer research questions.
+
+The model represents classes of operational activities for (1) environmental sampling,
+(2) biosample preparation for instrument analysis,
+(3) instrument-analysis runs, and
+(4) computational-analysis workflow executions.
+
+The model represents metadata for biosamples, studies, and their associated operational activities
+in such a way that researcher teams can leverage existing data in order to more easily answer research questions
+with minimal duplication of effort, by either obtaining answers from the data of existing studies, or by designing
+new studies that maximally leverage existing data.
+
+## About
+
+A <span class="term">Domain Vision Statement</span>[^1] is a short description (about one page)
+of the <span class="term">core domain</span> and the value it will bring, the "value proposition."
+Ignore those aspects that do not distinguish this domain model from others.
+Show how the domain model serves and balances diverse interests.
+Keep it narrow.
+Write this statement early and revise it as you gain new insight.
+
+It should be usable directly by the management and technical staff during all phases of development
+to guide resource allocation, to guide modeling choices, and to educate team members.
+
+## References
+
+[^1]: [Domain Vision Statement](https://files.polyneme.xyz/domain-vision-statement-1696514386.pdf), pp415-6, in Evans, Eric. Domain-Driven Design: Tackling Complexity in the Heart of Software. Boston: Addison-Wesley, 2004.

docs/stylesheets/extra.css

@@ -0,0 +1,3 @@
+span.term {
+    font-variant: small-caps ;
+}

mkdocs.yml

@@ -3,6 +3,8 @@ site_description: Documentation for the National Microbiome Data Collaborative (
 site_url: https://microbiomedata.github.io/nmdc-runtime/
 repo_url: https://github.com/microbiomedata/nmdc-runtime/
 edit_uri_template: 'blob/main/docs/{path}'
+extra_css:
+  - stylesheets/extra.css
 nav:
   - index.md
   - Tutorials:
@@ -46,6 +48,7 @@ theme:
 markdown_extensions:
   - admonition
   - attr_list
+  - footnotes
   - md_in_html
   - toc:
       permalink: true

nmdc_runtime/api/analytics.py

@@ -0,0 +1,64 @@
+"""
+Based on <https://github.com/tom-draper/api-analytics/tree/main/analytics/python/fastapi>
+under MIT License <https://github.com/tom-draper/api-analytics/blob/main/analytics/python/fastapi/LICENSE>
+"""
+
+from datetime import datetime
+import threading
+from time import time
+from typing import Dict, List
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.types import ASGIApp
+from toolz import merge
+
+from nmdc_runtime.api.db.mongo import get_mongo_db
+
+_requests = []
+_last_posted = datetime.now()
+
+
+def _post_requests(collection: str, requests_data: List[Dict], source: str):
+    mdb = get_mongo_db()
+    mdb[collection].insert_many([merge(d, {"source": source}) for d in requests_data])
+
+
+def log_request(collection: str, request_data: Dict, source: str = "FastAPI"):
+    global _requests, _last_posted
+    _requests.append(request_data)
+    now = datetime.now()
+    # flush queue every minute at most
+    if (now - _last_posted).total_seconds() > 60.0:
+        threading.Thread(
+            target=_post_requests, args=(collection, _requests, source)
+        ).start()
+        _requests = []
+        _last_posted = now
+
+
+class Analytics(BaseHTTPMiddleware):
+    def __init__(self, app: ASGIApp, collection: str = "_runtime.analytics"):
+        super().__init__(app)
+        self.collection = collection
+
+    async def dispatch(
+        self, request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        start = time()
+        response = await call_next(request)
+
+        request_data = {
+            "hostname": request.url.hostname,
+            "ip_address": request.client.host,
+            "path": request.url.path,
+            "user_agent": request.headers["user-agent"],
+            "method": request.method,
+            "status": response.status_code,
+            "response_time": int((time() - start) * 1000),
+            "created_at": datetime.now().isoformat(),
+        }
+
+        log_request(self.collection, request_data, "FastAPI")
+        return response

nmdc_runtime/api/main.py

@@ -9,6 +9,7 @@
 from starlette import status
 from starlette.responses import RedirectResponse
 
+from nmdc_runtime.api.analytics import Analytics
 from nmdc_runtime.util import all_docs_have_unique_id, ensure_unique_id_indexes
 from nmdc_runtime.api.core.auth import get_password_hash
 from nmdc_runtime.api.db.mongo import (
@@ -363,6 +364,7 @@ async def root():
     allow_methods=["*"],
     allow_headers=["*"],
 )
+app.add_middleware(Analytics)
 
 
 if __name__ == "__main__":