Adds function to easily store and retrieve dataframe through a REST API

_doc/sphinxdoc/source/api/index.rst

@@ -14,3 +14,4 @@ API Summary
     loghelper
     pycode
     texthelper
+    server

_doc/sphinxdoc/source/api/server.rst

@@ -0,0 +1,29 @@
+
+REST API, local file store
+==========================
+
+Benchmarking usually happens on a different job when running
+CI jobs and cannot be included in the documentation unless
+they are stored somewhere. A REST API is better than
+a local file because it can be distance and do not rely
+on local path. These functions are a simple implementation
+of an API to store and retrieve dataframes with :epkg:`FastAPI`.
+
+.. contents::
+    :local:
+
+REST API
+++++++++
+
+.. autosignature:: pyquickhelper.server.filestore_fastapi.fast_api_submit
+
+.. autosignature:: pyquickhelper.server.filestore_fastapi.fast_api_query
+
+.. autosignature:: pyquickhelper.server.filestore_fastapi.fast_api_content
+
+.. autosignature:: pyquickhelper.server.filestore_fastapi.create_app
+
+File Storage
+++++++++++++
+
+.. autosignature:: pyquickhelper.server.filestore_sqlapi.SqlLite3FileStore

_unittests/ut_serverdoc/test_file_store_rest.py

@@ -1,18 +1,33 @@
 """
-@brief      test log(time=1s)
+@brief      test log(time=4s)
 
 """
 import unittest
 import os
 import pandas
 from pyquickhelper.pycode import ExtTestCase, get_temp_folder
-from pyquickhelper.server.filestore_fastapi import create_fast_api_app
+from pyquickhelper.server.filestore_fastapi import (
+    create_fast_api_app, fast_api_submit, fast_api_query,
+    fast_api_content, _get_password, _post_request)
 from fastapi.testclient import TestClient  # pylint: disable=E0401
 from pyquickhelper.server.filestore_sqlite import SqlLite3FileStore
 
 
 class TestfileStoreRest(ExtTestCase):
 
+    def test_simple_function1(self):
+        self.assertRaise(
+            lambda: _get_password(None, "IMPOSSIBLE"), RuntimeError)
+
+    def test_simple_function2(self):
+        from requests.exceptions import ConnectionError
+        self.assertRaise(
+            lambda: _post_request(None, None, None, None), AttributeError)
+        self.assertRaise(
+            lambda: _post_request(None, "http://localhost:7777", {}, "submit",
+                                  timeout=1.),
+            ConnectionError)
+
     def test_file_store(self):
         temp = get_temp_folder(__file__, "temp_file_storage_rest")
         name = os.path.join(temp, "filestore.db3")
@@ -52,6 +67,12 @@ def test_file_store(self):
         js = response.json()
         self.assertEqual(len(js), 1)
 
+        response = client.post(
+            "/content/", json=dict(password="BBB"))
+        self.assertEqual(response.status_code, 200)
+        js = response.json()
+        self.assertEqual(len(js), 1)
+
         response = client.post(
             "/metrics/", json=dict(name="essai", password="CCC"))
         self.assertEqual(response.status_code, 401)
@@ -86,6 +107,53 @@ def test_file_store(self):
         self.assertEqual(len(js), 1)
         self.assertEqual(js[0]['value'], 0.67)
 
+    def test_file_store_df(self):
+        temp = get_temp_folder(__file__, "temp_file_storage_rest_df")
+        name = os.path.join(temp, "filestore.db3")
+        app = create_fast_api_app(name, "BBB")
+        client = TestClient(app)
+        response = client.get("/")
+        self.assertEqual(response.status_code, 200)
+
+        df = pandas.DataFrame(
+            dict(A=[0, 5, 6], B=[4.5, 1, 1], C=["E", "R", "T"]))
+        resp = fast_api_submit(df, client, team="AA", name="BB", project="CCC",
+                               version=1, password="BBB")
+        self.assertEqual(resp.status_code, 200)
+
+        df = pandas.DataFrame(
+            dict(A=[0, 5, 6], B=[4.5, 2, 2], C=["E", "R", "Z"]))
+        resp = fast_api_submit(df, client, team="AA", name="BB", project="CCC",
+                               version=2, password="BBB")
+        self.assertEqual(resp.status_code, 200)
+
+        res = fast_api_query(client, team="AA", name="BB", project="CCC",
+                             password="BBB")
+        exp = [{'id': 1, 'name': 'BB',
+                'metadata': {'client': ['testclient', 50000]}, 'team': 'AA',
+                'project': 'CCC', 'version': 1, 'format': 'df'},
+               {'id': 2, 'name': 'BB',
+                'metadata': {'client': ['testclient', 50000]}, 'team': 'AA',
+                'project': 'CCC', 'version': 2, 'format': 'df'}]
+        for r in res:
+            del r['date']
+        self.assertEqual(res, exp)
+
+        df = fast_api_query(client, team="AA", name="BB", project="CCC",
+                            password="BBB", as_df=True)
+        cols = ['team', 'project', 'name', 'version']
+        df = df[cols]
+        mv = df.groupby(cols[:-1]).max()
+        self.assertEqual(mv.shape, (1, 1))
+        self.assertEqual(mv.iloc[0, 0], 2)
+
+        content = fast_api_content(
+            client, team="AA", name="BB", project="CCC",
+            password="BBB", as_df=True)
+        for c in content:
+            self.assertIsInstance(c['content'], pandas.DataFrame)
+            self.assertEqual(c['content'].shape, (3, 3))
+
 
 if __name__ == "__main__":
     unittest.main()

src/pyquickhelper/helpgen/utils_sphinx_doc_helpers.py

@@ -1,9 +1,7 @@
 """
 @file
-@brief various variables and classes used to produce a Sphinx documentation
-
+@brief Various variables and classes used to produce a Sphinx documentation.
 """
-
 import inspect
 import os
 import copy
@@ -732,7 +730,6 @@ def import_module(rootm, filename, log_function, additional_sys_path=None,
     except SystemError as e:  # pragma: no cover
         log_function("[warning] -- unable to import module (2) ", filename,
                      ",", fi, " in path ", sdir, " Error: ", str(e))
-        import traceback
         stack = traceback.format_exc()
         log_function("      executable", sys.executable)
         log_function("      version", sys.version_info)
@@ -752,7 +749,6 @@ def import_module(rootm, filename, log_function, additional_sys_path=None,
         else:
             log_function("[warning] -- unable to import module (4) ", filename,
                          ",", fi, " in path ", sdir, " Error: ", str(e))
-            import traceback
             stack = traceback.format_exc()
             log_function("      executable", sys.executable)
             log_function("      version", sys.version_info)

src/pyquickhelper/loghelper/github_api.py

@@ -18,8 +18,8 @@ def __init__(self, response, url, **kwargs):
         if msg:
             msg = "\n" + "\n".join(msg)
         Exception.__init__(self,
-            "response={0}\nurl={1}\ntext={2}\nstatus={3}{4}".format(
-                response, url, response.text, response.status_code, msg))
+                           "response={0}\nurl={1}\ntext={2}\nstatus={3}{4}".format(
+                               response, url, response.text, response.status_code, msg))
 
 
 def call_github_api(owner, repo, ask, auth=None, headers=None):

src/pyquickhelper/server/filestore_fastapi.py

@@ -4,6 +4,7 @@
 @brief Simple class to store and retrieve files through an API.
 """
 import os
+import io
 from typing import Optional
 from fastapi import FastAPI, Request, HTTPException
 from pydantic import BaseModel  # pylint: disable=E0611
@@ -15,7 +16,7 @@ class Item(BaseModel):
     format: Optional[str]  # pylint: disable=E1136
     team: Optional[str]  # pylint: disable=E1136
     project: Optional[str]  # pylint: disable=E1136
-    version: Optional[str]  # pylint: disable=E1136
+    version: Optional[int]  # pylint: disable=E1136
     content: Optional[str]  # pylint: disable=E1136
     password: str
 
@@ -30,7 +31,16 @@ class Query(BaseModel):
     name: Optional[str]  # pylint: disable=E1136
     team: Optional[str]  # pylint: disable=E1136
     project: Optional[str]  # pylint: disable=E1136
-    version: Optional[str]  # pylint: disable=E1136
+    version: Optional[int]  # pylint: disable=E1136
+    password: str
+
+
+class QueryL(BaseModel):
+    name: Optional[str]  # pylint: disable=E1136
+    team: Optional[str]  # pylint: disable=E1136
+    project: Optional[str]  # pylint: disable=E1136
+    version: Optional[int]  # pylint: disable=E1136
+    limit: Optional[int]  # pylint: disable=E1136
     password: str
 
 
@@ -71,9 +81,32 @@ async def query(query: Query, request: Request):
                                    project=query.project, version=query.version))
         return res
 
+    async def content(query: QueryL, request: Request):
+        if query.password != password:
+            raise HTTPException(status_code=401, detail="Wrong password")
+        if query.limit is None:
+            limit = 5
+        else:
+            limit = query.limit
+        res = []
+        for r in store.enumerate_content(
+                name=query.name, team=query.team, project=query.project,
+                version=query.version):
+            if len(res) >= limit:
+                break
+            if "content" in r:
+                content = r['content']
+                if hasattr(content, 'to_csv'):
+                    st = io.StringIO()
+                    content.to_csv(st, index=False, encoding="utf-8")
+                    r['content'] = st.getvalue()
+            res.append(r)
+        return res
+
     app = FastAPI()
     app.get("/")(get_root)
     app.post("/submit/")(submit)
+    app.post("/content/")(content)
     app.post("/metrics/")(metrics)
     app.post("/query/")(query)
     return app
@@ -124,3 +157,114 @@ def create_app():
     app = create_fast_api_app(os.environ['PYQUICKHELPER_FASTAPI_PATH'],
                               os.environ['PYQUICKHELPER_FASTAPI_PWD'])
     return app
+
+
+def _get_password(password, env="PYQUICKHELPER_FASTAPI_PWD"):
+    if password is None:
+        password = os.environ.get(env, None)
+    if password is None:
+        raise RuntimeError(
+            "password must be specified or environement variable "
+            "'PYQUICKHELPER_FASTAPI_PWD'.")
+    return password
+
+
+def _post_request(client, url, data, suffix, timeout=None):
+    if client is None:
+        import requests
+        resp = requests.post("%s/%s" % (url.strip('/'), suffix), data=data,
+                             timeout=timeout)
+    else:
+        resp = client.post("/%s/" % suffix, json=data)
+    if resp.status_code != 200:
+        del data['content']
+        del data['password']
+        raise RuntimeError(
+            "Post request failed due to %r\ndata=%r." % (resp, data))
+    return resp
+
+
+def fast_api_submit(df, client=None, url=None, name=None, team=None,
+                    project=None, version=None, password=None):
+    """
+    Stores a dataframe into a local stores.
+
+    :param df: dataframe
+    :param client: for unittest purpose
+    :param url: API url (can be None if client is not)
+    :param name: name
+    :param team: team
+    :param project: project
+    :param version: version
+    :param password: password for the submission
+    :return: response
+    """
+    password = _get_password(password)
+    st = io.StringIO()
+    df.to_csv(st, index=False, encoding="utf-8")
+    data = dict(team=team, project=project, version=version,
+                password=password, content=st.getvalue(),
+                name=name, format="df")
+    return _post_request(client, url, data, "submit")
+
+
+def fast_api_query(client=None, url=None, name=None, team=None,
+                   project=None, version=None, password=None,
+                   as_df=False):
+    """
+    Retrieves the list of dataframe based on partial information.
+
+    :param client: for unittest purpose
+    :param url: API url (can be None if client is not)
+    :param name: name
+    :param team: team
+    :param project: project
+    :param version: version
+    :param password: password for the submission
+    :return: response
+    """
+    password = _get_password(password)
+    data = dict(team=team, project=project, version=version,
+                password=password, name=name)
+    resp = _post_request(client, url, data, "query")
+    if as_df:
+        import pandas
+        return pandas.DataFrame(resp.json())
+    return resp.json()
+
+
+def fast_api_content(client=None, url=None, name=None, team=None,
+                     project=None, version=None, limit=5,
+                     password=None, as_df=True):
+    """
+    Retrieves the dataframes based on partial information.
+    Enumerates a list of dataframes.
+
+    :param client: for unittest purpose
+    :param url: API url (can be None if client is not)
+    :param name: name
+    :param team: team
+    :param project: project
+    :param version: version
+    :param limit: maximum number of dataframes to retrieve
+    :param as_df: returns the content as a dataframe
+    :param password: password for the submission
+    :return: list of dictionary, content is a dataframe
+    """
+    password = _get_password(password)
+    data = dict(team=team, project=project, version=version,
+                password=password, name=name, limit=limit)
+    resp = _post_request(client, url, data, "content")
+    res = resp.json()
+    if as_df:
+        import pandas
+
+        for r in res:
+            content = r.get('content', None)
+            if content is None:
+                continue
+            if 'format' in r and r['format'] == 'df':
+                st = io.StringIO(r['content'])
+                df = pandas.read_csv(st, encoding="utf-8")
+                r['content'] = df
+    return res