Adds class to store files with sqlite3, access with FastAPI

_unittests/ut_serverdoc/test_file_store.py

@@ -0,0 +1,70 @@
+"""
+@brief      test log(time=1s)
+
+"""
+import unittest
+import os
+import pandas
+from pyquickhelper.pycode import ExtTestCase, get_temp_folder
+from pyquickhelper.server.filestore_sqlite import SqlLite3FileStore
+
+
+class TestfileStore(ExtTestCase):
+
+    def test_file_store(self):
+        temp = get_temp_folder(__file__, "temp_file_storage")
+        name = os.path.join(temp, "filestore.db3")
+        store = SqlLite3FileStore(name)
+        df = pandas.DataFrame({"A": ["un", "deux"], "B": [0.5, 0.6]})
+        store.add(name="zoo", metadata={'hh': 'kk'}, content=df)
+        got = list(store.enumerate_content(name="zoo"))
+        self.assertEqual(len(got), 1)
+        record = got[0]
+        name = record['name']
+        self.assertEqual(name, "zoo")
+        self.assertIn("date", record)
+        content = record['content']
+        self.assertIsInstance(df, pandas.DataFrame)
+        self.assertEqualDataFrame(df, content)
+        meta = record['metadata']
+        self.assertIsInstance(meta, dict)
+        self.assertEqual(meta, {'hh': 'kk'})
+        got = list(store.enumerate(name="zoo"))
+        self.assertEqual(len(got), 1)
+
+        # data
+        idfile = record['id']
+        store.add_data(idfile=idfile, name="ZOO", value="5.6")
+        res = list(store.enumerate_data(idfile))
+        self.assertEqual(len(res), 1)
+        del res[0]['date']
+        self.assertEqual(res, [{'id': 1, 'idfile': 1, 'name': 'ZOO',
+                                'value': 5.6}])
+
+        # data join
+        res = list(store.enumerate_data(idfile, join=True))
+        self.assertEqual(len(res), 1)
+        del res[0]['date']
+        self.assertEqual(
+            res, [{'id': 1, 'idfile': 1, 'name': 'ZOO',
+                   'name_f': 'zoo', 'value': 5.6}])
+
+    def test_file_store_exc(self):
+        temp = get_temp_folder(__file__, "temp_file_storage_exc")
+        name = os.path.join(temp, "filestore.db3")
+        store = SqlLite3FileStore(name)
+        df = pandas.DataFrame({"A": ["un", "deux"], "B": [0.5, 0.6]})
+        self.assertRaise(
+            lambda: store.add(name="zoo", metadata="{'hh': 'kk'}",
+                              content=df),
+            TypeError)
+
+    def test_file_store1(self):
+        temp = get_temp_folder(__file__, "temp_file_storage")
+        name = os.path.join(temp, "filestore.db3")
+        SqlLite3FileStore(name)
+        SqlLite3FileStore(name)
+
+
+if __name__ == "__main__":
+    unittest.main()

_unittests/ut_serverdoc/test_file_store_rest.py

@@ -0,0 +1,60 @@
+"""
+@brief      test log(time=1s)
+
+"""
+import unittest
+import os
+import pandas
+from pyquickhelper.pycode import ExtTestCase, get_temp_folder
+from pyquickhelper.server.filestore_fastapi import create_fast_api_app
+from fastapi.testclient import TestClient  # pylint: disable=E0401
+
+
+class TestfileStoreRest(ExtTestCase):
+
+    def test_file_store(self):
+        temp = get_temp_folder(__file__, "temp_file_storage_rest")
+        name = os.path.join(temp, "filestore.db3")
+        app = create_fast_api_app(name, "BBB")
+        client = TestClient(app)
+        response = client.get("/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(
+            response.json(),
+            {'pyquickhelper': 'FastAPI to load and query files'})
+        response = client.post(
+            "/add/", json=dict(name="essai", content="a,b\ne,0\nhh, 1.5",
+                               password="CCC"))
+        self.assertEqual(response.status_code, 401)
+
+        response = client.post(
+            "/add/", json=dict(name="essai", content="a,b\ne,0\nhh, 1.5",
+                               password="BBB"))
+        self.assertEqual(response.status_code, 200)
+        js = response.json()
+        self.assertIn('date', js)
+        self.assertNotIn('content', js)
+
+        response = client.post(
+            "/query/", json=dict(name="essai", password="CCC"))
+        self.assertEqual(response.status_code, 401)
+
+        response = client.post(
+            "/query/", json=dict(name="essai", password="BBB"))
+        self.assertEqual(response.status_code, 200)
+        js = response.json()
+        self.assertEqual(len(js), 1)
+
+        response = client.post(
+            "/metrics/", json=dict(name="essai", password="CCC"))
+        self.assertEqual(response.status_code, 401)
+
+        response = client.post(
+            "/metrics/", json=dict(name="essai", password="BBB"))
+        self.assertEqual(response.status_code, 200)
+        js = response.json()
+        self.assertEqual(len(js), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

requirements.txt

@@ -6,6 +6,7 @@ codecov
 coverage>=5.0
 cryptography
 docformatter
+fastapi
 fire
 git-pandas
 gitdb

setup.py

@@ -266,6 +266,7 @@ def write_version():
                            'keyrings.cryptfile'],
             'jenkinshelper': ['python-jenkins>=1.0.0', 'pyyaml'],
             'loghelper': ['psutil'],
+            'server': ['fastapi'],
             'all': [
                 "autopep8",     # part of the minimal list
                 'cffi',
@@ -274,6 +275,7 @@ def write_version():
                 "docformatter",
                 "docutils",
                 'flake8',
+                'fastapi',
                 'fire',
                 "IPython>=5.0.0",
                 "jupyter",

src/pyquickhelper/__main__.py

@@ -31,6 +31,7 @@ def main(args, fLOG=print):
         from .cli.notebook import run_notebook, convert_notebook
         from .loghelper import set_password
         from .filehelper.download_urls_helper import download_urls_in_folder_content
+        from .cli.uvicorn_cli import uvicorn_app
     except ImportError:  # pragma: no cover
         from pyquickhelper.cli.pyq_sync_cli import pyq_sync
         from pyquickhelper.cli.encryption_file_cli import encrypt_file, decrypt_file
@@ -47,6 +48,7 @@ def main(args, fLOG=print):
         from pyquickhelper.cli.notebook import run_notebook, convert_notebook
         from pyquickhelper.loghelper import set_password
         from pyquickhelper.filehelper.download_urls_helper import download_urls_in_folder_content
+        from pyquickhelper.cli.uvicorn_cli import uvicorn_app
 
     fcts = dict(synchronize_folder=pyq_sync, encrypt_file=encrypt_file,
                 decrypt_file=decrypt_file, encrypt=encrypt,
@@ -58,7 +60,8 @@ def main(args, fLOG=print):
                 zoom_img=zoom_img, images2pdf=images2pdf,
                 repeat_script=repeat_script,
                 ftp_upload=ftp_upload, set_password=set_password,
-                download_urls_in_folder_content=download_urls_in_folder_content)
+                download_urls_in_folder_content=download_urls_in_folder_content,
+                uvicorn_app=uvicorn_app)
     return cli_main_helper(fcts, args=args, fLOG=fLOG)
 
 

src/pyquickhelper/cli/uvicorn_cli.py

@@ -0,0 +1,31 @@
+"""
+@file
+@brief Simplified function versions.
+"""
+import os
+
+
+def uvicorn_app(path="dummy_db.db3", pwd="dummy", port=8798, host="127.0.0.1"):
+    """
+    Runs a uvicorn application. It should be used for testing
+    not for production. Use ``host:post/redoc`` or
+    ``host:post/docs`` to get a web page in order to
+    submit files.
+
+    :param path: filename for the databse
+    :param pwd: password
+    :param host: host
+    :param port: port
+
+    .. cmdref::
+        :title: Runs a uvicorn application
+        :cmd: -m pyquickhelper uvicorn_app --help
+
+        Runs a uvicorn application.
+    """
+    from ..server.filestore_fastapi import create_app  # pylint: disable=W0611
+    import uvicorn
+    os.environ['PYQUICKHELPER_FASTAPI_PWD'] = pwd
+    os.environ['PYQUICKHELPER_FASTAPI_PATH'] = path
+    uvicorn.run("pyquickhelper.server.filestore_fastapi:create_app",
+                host=host, port=port, log_level="info", factory=True)

src/pyquickhelper/helpgen/default_conf.py

@@ -94,6 +94,7 @@ def get_epkg_dictionary():
         'django': 'https://www.djangoproject.com/',
         'docutils': 'http://docutils.sourceforge.net/',
         'dvipng': 'https://ctan.org/pkg/dvipng?lang=en',
+        'FastAPI': 'https://fastapi.tiangolo.com/',
         'format style': 'https://pyformat.info/>`_',
         'FTP': 'https://en.wikipedia.org/wiki/File_Transfer_Protocol',
         'getsitepackages': 'https://docs.python.org/3/library/site.html#site.getsitepackages',
@@ -188,6 +189,7 @@ def get_epkg_dictionary():
         'Python': 'http://www.python.org/',
         'python-jenkins': 'http://python-jenkins.readthedocs.org/en/latest/',
         'pywin32': 'https://sourceforge.net/projects/pywin32/',
+        'REST': 'https://en.wikipedia.org/wiki/Representational_state_transfer',
         'reveal.js': 'https://github.com/hakimel/reveal.js/releases',
         'rst': 'https://en.wikipedia.org/wiki/ReStructuredText',
         'RST': 'https://en.wikipedia.org/wiki/ReStructuredText',
@@ -218,6 +220,7 @@ def get_epkg_dictionary():
         'tornado': 'http://www.tornadoweb.org/en/stable/',
         'TortoiseSVN': 'http://tortoisesvn.net/',
         'travis': 'https://travis-ci.org/',
+        'uvicorn': 'https://www.uvicorn.org/',
         'vis.js': 'https://visjs.org/',
         'viz.js': 'https://github.com/mdaines/viz.js/',
         'Visual Studio Community Edition 2015': 'https://imagine.microsoft.com/en-us/Catalog/Product/101',

src/pyquickhelper/ipythonhelper/magic_class_diff.py

@@ -77,7 +77,8 @@ def textdiff(self, line):
         if args is not None:
             html, js = create_visual_diff_through_html_files(
                 args.f1, args.f2, encoding=args.encoding, notebook=True,
-                context_size=None if args.context in [None, ""] else int(args.context),
+                context_size=None if args.context in [
+                    None, ""] else int(args.context),
                 inline_view=args.inline)
             display_html(html)
             return js

src/pyquickhelper/server/filestore_fastapi.py

@@ -0,0 +1,114 @@
+# -*- coding:utf-8 -*-
+"""
+@file
+@brief Simple class to store and retrieve files through an API.
+"""
+import os
+from typing import Optional
+from fastapi import FastAPI, Request, HTTPException
+from pydantic import BaseModel  # pylint: disable=E0611
+from .filestore_sqlite import SqlLite3FileStore
+
+
+class Item(BaseModel):
+    name: Optional[str]  # pylint: disable=E1136
+    format: Optional[str]  # pylint: disable=E1136
+    team: Optional[str]  # pylint: disable=E1136
+    project: Optional[str]  # pylint: disable=E1136
+    version: Optional[str]  # pylint: disable=E1136
+    content: Optional[str]  # pylint: disable=E1136
+    password: str
+
+
+class Metric(BaseModel):
+    name: str
+    password: str
+
+
+class Query(BaseModel):
+    name: Optional[str]  # pylint: disable=E1136
+    team: Optional[str]  # pylint: disable=E1136
+    project: Optional[str]  # pylint: disable=E1136
+    version: Optional[str]  # pylint: disable=E1136
+    password: str
+
+
+def create_fast_api_app(db_path, password):
+    """
+    Creates a :epkg:`REST` application based on :epkg:`FastAPI`.
+
+    :return: app
+    """
+    store = SqlLite3FileStore(db_path)
+
+    async def get_root():
+        return {"pyquickhelper": "FastAPI to load and query files"}
+
+    async def add(item: Item, request: Request):
+        if item.password != password:
+            raise HTTPException(status_code=401, detail="Wrong password")
+        kwargs = dict(name=item.name, format=item.format,
+                      team=item.team, project=item.project,
+                      version=item.version, content=item.content)
+        kwargs['metadata'] = dict(client=request.client)
+        res = store.add(**kwargs)
+        if 'content' in res:
+            del res['content']
+        return res
+
+    async def metrics(query: Metric, request: Request):
+        if query.password != password:
+            raise HTTPException(status_code=401, detail="Wrong password")
+        res = list(store.enumerate_data(name=query.name, join=True))
+        return res
+
+    async def query(query: Query, request: Request):
+        if query.password != password:
+            raise HTTPException(status_code=401, detail="Wrong password")
+        res = list(store.enumerate(name=query.name, team=query.team,
+                                   project=query.project, version=query.version))
+        return res
+
+    app = FastAPI()
+    app.get("/")(get_root)
+    app.post("/add/")(add)
+    app.post("/metrics/")(metrics)
+    app.post("/query/")(query)
+    return app
+
+
+def create_app():
+    """
+    Creates an instance of application class returned
+    by @see fn create_fast_api_app. It checks that
+    environment variables ``PYQUICKHELPER_FASTAPI_PWD``
+    and ``PYQUICKHELPER_FASTAPI_PATH`` are set up with
+    a password and a filename. Otherwise, the function
+    raised an exception.
+
+    Inspired from the guidelines
+    `uvicorn/deployment <https://www.uvicorn.org/deployment/>`_,
+    `(2) <https://www.uvicorn.org/deployment/#running-programmatically>`_.
+    Some command lines:
+
+    ::
+
+        uvicorn --factory pyquickhelper.server.filestore_fastapi:create_app --port 8798
+                --ssl-keyfile=./key.pem --ssl-certfile=./cert.pem
+        gunicorn --keyfile=./key.pem --certfile=./cert.pem -k uvicorn.workers.UvicornWorker
+                 --factory pyquickhelper.server.filestore_fastapi:create_app
+
+    ::
+
+        uvicorn.run("pyquickhelper.server.filestore_fastapi:create_app",
+                    host="127.0.0.1", port=8798, log_level="info", factory=True)
+    """
+    if "PYQUICKHELPER_FASTAPI_PWD" not in os.environ:
+        raise RuntimeError(
+            "Environment variable PYQUICKHELPER_FASTAPI_PWD is missing.")
+    if "PYQUICKHELPER_FASTAPI_PATH" not in os.environ:
+        raise RuntimeError(
+            "Environment variable PYQUICKHELPER_FASTAPI_PATH is missing.")
+    app = create_fast_api_app(os.environ['PYQUICKHELPER_FASTAPI_PATH'],
+                              os.environ['PYQUICKHELPER_FASTAPI_PWD'])
+    return app