automatic documentation for application apis

Dockerfile

@@ -4,7 +4,7 @@ RUN apt-get update && apt-get install -y git
 
 #RUN pip install apihub
 COPY . /code
-RUN pip install /code/dist/apihub-0.1.0-py3-none-any.whl
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
 
 # expose port for prometheus
 EXPOSE 8000
@@ -14,4 +14,4 @@ EXPOSE 5000
 
 ENV PORT 5000
 
-CMD ["apihub_server"]
+CMD ["poetry", "apihub_server"]

Makefile

@@ -19,6 +19,25 @@ pdb:
 mypy: 
 	poetry run mypy --ignore-missing-imports apihub
 
+.PHONY: result
+result:
+	PYTHONPATH=../pipeline/src poetry run python apihub/result.py \
+		--in-kind FILE --in-filename tests/fixtures/result_input.txt \
+		--out-kind FILE --out-filename - \
+		--debug
+
+.PHONY: redis-result
+redis-result:
+	PYTHONPATH=../pipeline/src poetry run python apihub/result.py \
+	  --in-kind LREDIS --in-redis redis://localhost:6379/1 \
+		--in-topic result --in-namespace apihub \
+		--debug
+
+.PHONY: server
+server:
+	PYTHONPATH=../pipeline/src poetry run python apihub/server.py \
+		--out-kind LREDIS --debug
+
 .PHONY: pre-commit
 pre-commit: 
 	pre-commit run --all-files

README.rst

@@ -20,7 +20,7 @@
    <br />
     <p align="center">
       <a href="https://github.com/yifan/apihub">
-        <img src="https://raw.githubusercontent.com/yifan/apihub/master/images/APIHub.png" alt="Logo" width="600" height="400">
+        <img src="https://raw.githubusercontent.com/yifan/apihub/master/images/APIHub-logo.png" alt="Logo" width="200" height="100">
       </a>
 
       <h3 align="center">APIHub</h3>
@@ -89,32 +89,26 @@ About The Project
 `[Product Name Screen
 Shot][product-screenshot] <https://raw.githubusercontent.com/yifan/apihub/master/images/APIHub.png>`__
 
-Here’s a blank template to get started: **To avoid retyping too much
-info. Do a search and replace with your text editor for the following:**
-``yifan``, ``apihub``, ``yifan2019``, ``email``, ``APIHub``,
-``project_description``
+
 
 Features & TODOs
 ----------------
 
-::
-
-   [X] Security
-       [X] authenticate
-       [X] admin, manager, user
-       [X] user management
-       [X] rate limiter
-       [ ] register
-       [ ] social login
-   [ ] Subscription
-       [-] subscription
-       [-] quota
-       [X] application token
-       [-] daily usage record in redis
-   [ ] Async/sync API calls
-       [ ] api worker reports input/output: describe
-       [X] generic worker deployment 
-       [ ] auto scaler for api workers
+- |check| Security
+    |check| authenticate
+    |check| admin, manager, user
+    |check| user management
+  -    |check| rate limiter
+  -    |check| register
+  -    |uncheck_| social login
+-  |check| Subscription
+  -    |check| subscription
+  -    |check| application token
+  -    |check| daily usage record in redis
+-  |uncheck| Async/sync API calls
+  -    |check| api worker reports input/output: describe
+  -    |check| generic worker deployment 
+  -    |uncheck| auto scaler for api workers
 
 Built With
 ----------
@@ -276,6 +270,22 @@ Copyright (C) 2021, Qatar Computing Research Institute, HBKU
 
    <!-- https://www.markdownguide.org/basic-syntax/#reference-style-links -->
 
+.. |check| raw:: html
+
+    <input checked=""  type="checkbox">
+
+.. |check_| raw:: html
+
+    <input checked=""  disabled="" type="checkbox">
+
+.. |uncheck| raw:: html
+
+    <input type="checkbox">
+
+.. |uncheck_| raw:: html
+
+    <input disabled="" type="checkbox">
+
 
 .. |Contributors| image:: https://img.shields.io/github/contributors/yifan/apihub.svg?style=for-the-badge
    :target: https://github.com/yifan/apihub/graphs/contributors

apihub/cli.py

@@ -91,7 +91,7 @@ def create_subscription(admin, limit, days, recurring, username, application):
             "username": username,
             "application": application,
             "starts_at": datetime.now(),
-            "limit": limit,
+            "credit": limit,
             "expires_at": expires_at,
             "recurring": recurring,
         }

apihub/result.py

@@ -1,10 +1,8 @@
-import json
-
 import redis
 from prometheus_client import Counter, Histogram
 from dotenv import load_dotenv
 
-from pipeline import ProcessorSettings, Processor, Command, CommandActions
+from pipeline import ProcessorSettings, Processor, Command, CommandActions, Definition
 from apihub.utils import Result, Status, RedisSettings, DEFINITION
 from apihub import __worker__, __version__
 
@@ -44,12 +42,17 @@ def setup(self) -> None:
         self.redis = redis.Redis.from_url(settings.redis)
 
     def process_command(self, command: Command) -> None:
+        self.logger.info("Processing COMMAND")
         if command.action == CommandActions.Define:
-            for k, v in command.content.items():
-                self.redis.hset(DEFINITION, k, json.dumps(v))
-                self.logger.info(f"{k} definition:\n{json.dumps(v, indent=2)}")
+            definition = Definition.parse_obj(command.content)
+            self.logger.info(definition)
+            self.redis.hset(DEFINITION, definition.source.topic, definition.json())
+            self.logger.info(
+                f"{definition.source.topic} definition:\n{definition.json()}"
+            )
 
     def process(self, message_content, message_id):
+        self.logger.info("Processing MESSAGE")
         result = Result.parse_obj(message_content)
         if result.status == Status.PROCESSED:
             result.result = {

apihub/server.py

@@ -1,20 +1,22 @@
 import sys
 import functools
+import logging
 from typing import Dict, Any
 
 from fastapi import FastAPI, HTTPException, Request, Query, Depends
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field
 from fastapi_jwt_auth import AuthJWT
 from fastapi_jwt_auth.exceptions import AuthJWTException
+from fastapi.openapi.utils import get_openapi
 from dotenv import load_dotenv
-from pipeline import Message, Settings, Command, CommandActions, Monitor
+from pipeline import Message, Settings, Command, CommandActions, Monitor, Definition
 from apihub_users.security.depends import RateLimiter, RateLimits, require_user
 from apihub_users.security.router import router as security_router
 from apihub_users.subscription.depends import require_subscription
 from apihub_users.subscription.router import router as application_router
 
-from apihub.utils import State, make_topic, make_key, Result, Status
+from apihub.utils import DEFINITION, State, make_topic, make_key, Result, Status
 from apihub import __worker__, __version__
 
 
@@ -30,7 +32,8 @@
 
 @functools.lru_cache(maxsize=None)
 def get_state():
-    return State()
+    logging.basicConfig(level=logging.DEBUG)
+    return State(logger=logging)
 
 
 def get_redis():
@@ -60,10 +63,12 @@ def jwt_get_config():
 api.include_router(
     security_router,
     #     prefix='/security',
-    #     tags=['security'],
+    tags=["security"],
     dependencies=[Depends(ip_rate_limited)],
 )
-api.include_router(application_router, dependencies=[Depends(ip_rate_limited)])
+api.include_router(
+    application_router, tags=["subscription"], dependencies=[Depends(ip_rate_limited)]
+)
 
 
 @api.exception_handler(AuthJWTException)
@@ -91,6 +96,7 @@ async def root():
 
 @api.get(
     "/define/{application}",
+    include_in_schema=False,
     dependencies=[Depends(ip_rate_limited)],
 )
 async def define_service(
@@ -101,9 +107,12 @@ async def define_service(
 
     get_state().write(make_topic(application), Command(action=CommandActions.Define))
 
+    return {"define": f"application {application}"}
+
 
 @api.post(
     "/async/{application}",
+    include_in_schema=False,
     response_model=AsyncAPIRequestResponse,
     dependencies=[Depends(ip_rate_limited)],
 )
@@ -144,7 +153,11 @@ async def async_service(
     return AsyncAPIRequestResponse(success=True, key=key)
 
 
-@api.get("/async/{application}", dependencies=[Depends(ip_rate_limited)])
+@api.get(
+    "/async/{application}",
+    dependencies=[Depends(ip_rate_limited)],
+    include_in_schema=False,
+)
 async def async_service_result(
     application: str,
     key: str = Query(
@@ -194,13 +207,101 @@ async def async_service_result(
     )
 
 
-@api.post("/sync/{service_name}")
+@api.post(
+    "/sync/{service_name}",
+    include_in_schema=False,
+)
 async def sync_service(service_name: str):
     """generic synchronised api hendler"""
     # TODO synchronised service, basically it will wait and return results
     # when it is ready. It will have a timeout of 30 seconds
 
 
+def get_paths(redis=get_redis()):
+    paths = {}
+    for name, dct_str in redis.hgetall(DEFINITION).items():
+        name = name.decode("utf-8")
+        path = {}
+        definition = Definition.parse_raw(dct_str)
+        operation = {}
+        operation["tags"] = ["app"]
+        operation["summary"] = definition.description
+        operation["description"] = definition.description
+        parameters = {}
+        operation["requestBody"] = {
+            "content": {
+                "application/json": {
+                    "schema": definition.input_schema,  # ["properties"],
+                }
+            },
+            "required": True,
+        }
+        operation["responses"] = {
+            "200": {
+                "description": "successful request",
+                "status_code": 200,
+                "content": {
+                    "application/json": {"schema": AsyncAPIRequestResponse.schema()}
+                },
+            }
+        }
+        path["post"] = operation
+
+        operation = {}
+        operation["tags"] = ["app"]
+        operation["summary"] = "obtain results from previous post requests"
+        operation["description"] = definition.description
+        parameters = [
+            {
+                "name": "key",
+                "in": "query",
+                "description": "the unique key obtained from post request",
+                "required": True,
+                "type": "string",
+                "format": "string",
+            }
+        ]
+        # a strange way to sort parameters maybe
+        operation["parameters"] = list(
+            {param["name"]: param for param in parameters}.values()
+        )
+        operation["responses"] = {
+            "200": {
+                "description": "success",
+                "content": {
+                    "application/json": {
+                        "schema": definition.output_schema,
+                    }
+                },
+            }
+        }
+        path["get"] = operation
+
+        paths[f"/async/{name}"] = path
+    return paths
+
+
+def custom_openapi(app=api):
+    # if app.openapi_schema:
+    #    return app.openapi_schema
+
+    openapi_schema = get_openapi(
+        title="APIHub",
+        version="0.1.0",
+        description="API for AI",
+        routes=app.routes,
+    )
+    openapi_schema["info"]["x-logo"] = {
+        "url": "https://raw.githubusercontent.com/yifan/apihub/master/images/APIHub-logo.png"
+    }
+    openapi_schema["paths"].update(get_paths())
+    app.openapi_schema = openapi_schema
+    return app.openapi_schema
+
+
+api.openapi = custom_openapi
+
+
 class ServerSettings(Settings):
     port: int = 5000
     log_level: str = "debug"

apihub/utils.py

@@ -30,8 +30,8 @@ class RedisSettings(Settings):
 
 
 class State:
-    def __init__(self):
-        self.pipeline = Pipeline()
+    def __init__(self, logger):
+        self.pipeline = Pipeline(logger=logger)
         settings = RedisSettings()
         settings.parse_args(args=[])
         self.redis = redis.Redis.from_url(settings.redis)
@@ -49,4 +49,4 @@ def make_key():
 
 
 def make_topic(service_name: str):
-    return f"api-{service_name}"
+    return service_name