diff --git a/_unittests/ut_serverdoc/test_file_store.py b/_unittests/ut_serverdoc/test_file_store.py
new file mode 100644
index 000000000..b8e6c605a
--- /dev/null
+++ b/_unittests/ut_serverdoc/test_file_store.py
@@ -0,0 +1,70 @@
+"""
+@brief test log(time=1s)
+
+"""
+import unittest
+import os
+import pandas
+from pyquickhelper.pycode import ExtTestCase, get_temp_folder
+from pyquickhelper.server.filestore_sqlite import SqlLite3FileStore
+
+
+class TestfileStore(ExtTestCase):
+
+ def test_file_store(self):
+ temp = get_temp_folder(__file__, "temp_file_storage")
+ name = os.path.join(temp, "filestore.db3")
+ store = SqlLite3FileStore(name)
+ df = pandas.DataFrame({"A": ["un", "deux"], "B": [0.5, 0.6]})
+ store.add(name="zoo", metadata={'hh': 'kk'}, content=df)
+ got = list(store.enumerate_content(name="zoo"))
+ self.assertEqual(len(got), 1)
+ record = got[0]
+ name = record['name']
+ self.assertEqual(name, "zoo")
+ self.assertIn("date", record)
+ content = record['content']
+ self.assertIsInstance(df, pandas.DataFrame)
+ self.assertEqualDataFrame(df, content)
+ meta = record['metadata']
+ self.assertIsInstance(meta, dict)
+ self.assertEqual(meta, {'hh': 'kk'})
+ got = list(store.enumerate(name="zoo"))
+ self.assertEqual(len(got), 1)
+
+ # data
+ idfile = record['id']
+ store.add_data(idfile=idfile, name="ZOO", value="5.6")
+ res = list(store.enumerate_data(idfile))
+ self.assertEqual(len(res), 1)
+ del res[0]['date']
+ self.assertEqual(res, [{'id': 1, 'idfile': 1, 'name': 'ZOO',
+ 'value': 5.6}])
+
+ # data join
+ res = list(store.enumerate_data(idfile, join=True))
+ self.assertEqual(len(res), 1)
+ del res[0]['date']
+ self.assertEqual(
+ res, [{'id': 1, 'idfile': 1, 'name': 'ZOO',
+ 'name_f': 'zoo', 'value': 5.6}])
+
+ def test_file_store_exc(self):
+ temp = get_temp_folder(__file__, "temp_file_storage_exc")
+ name = os.path.join(temp, "filestore.db3")
+ store = SqlLite3FileStore(name)
+ df = pandas.DataFrame({"A": ["un", "deux"], "B": [0.5, 0.6]})
+ self.assertRaise(
+ lambda: store.add(name="zoo", metadata="{'hh': 'kk'}",
+ content=df),
+ TypeError)
+
+ def test_file_store1(self):
+ temp = get_temp_folder(__file__, "temp_file_storage")
+ name = os.path.join(temp, "filestore.db3")
+ SqlLite3FileStore(name)
+ SqlLite3FileStore(name)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/_unittests/ut_serverdoc/test_file_store_rest.py b/_unittests/ut_serverdoc/test_file_store_rest.py
new file mode 100644
index 000000000..1154dc55d
--- /dev/null
+++ b/_unittests/ut_serverdoc/test_file_store_rest.py
@@ -0,0 +1,60 @@
+"""
+@brief test log(time=1s)
+
+"""
+import unittest
+import os
+import pandas
+from pyquickhelper.pycode import ExtTestCase, get_temp_folder
+from pyquickhelper.server.filestore_fastapi import create_fast_api_app
+from fastapi.testclient import TestClient # pylint: disable=E0401
+
+
+class TestfileStoreRest(ExtTestCase):
+
+ def test_file_store(self):
+ temp = get_temp_folder(__file__, "temp_file_storage_rest")
+ name = os.path.join(temp, "filestore.db3")
+ app = create_fast_api_app(name, "BBB")
+ client = TestClient(app)
+ response = client.get("/")
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(
+ response.json(),
+ {'pyquickhelper': 'FastAPI to load and query files'})
+ response = client.post(
+ "/add/", json=dict(name="essai", content="a,b\ne,0\nhh, 1.5",
+ password="CCC"))
+ self.assertEqual(response.status_code, 401)
+
+ response = client.post(
+ "/add/", json=dict(name="essai", content="a,b\ne,0\nhh, 1.5",
+ password="BBB"))
+ self.assertEqual(response.status_code, 200)
+ js = response.json()
+ self.assertIn('date', js)
+ self.assertNotIn('content', js)
+
+ response = client.post(
+ "/query/", json=dict(name="essai", password="CCC"))
+ self.assertEqual(response.status_code, 401)
+
+ response = client.post(
+ "/query/", json=dict(name="essai", password="BBB"))
+ self.assertEqual(response.status_code, 200)
+ js = response.json()
+ self.assertEqual(len(js), 1)
+
+ response = client.post(
+ "/metrics/", json=dict(name="essai", password="CCC"))
+ self.assertEqual(response.status_code, 401)
+
+ response = client.post(
+ "/metrics/", json=dict(name="essai", password="BBB"))
+ self.assertEqual(response.status_code, 200)
+ js = response.json()
+ self.assertEqual(len(js), 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/requirements.txt b/requirements.txt
index 0d5718a11..37352eda0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,6 +6,7 @@ codecov
coverage>=5.0
cryptography
docformatter
+fastapi
fire
git-pandas
gitdb
diff --git a/setup.py b/setup.py
index cb00507a5..99d54ca1e 100644
--- a/setup.py
+++ b/setup.py
@@ -266,6 +266,7 @@ def write_version():
'keyrings.cryptfile'],
'jenkinshelper': ['python-jenkins>=1.0.0', 'pyyaml'],
'loghelper': ['psutil'],
+ 'server': ['fastapi'],
'all': [
"autopep8", # part of the minimal list
'cffi',
@@ -274,6 +275,7 @@ def write_version():
"docformatter",
"docutils",
'flake8',
+ 'fastapi',
'fire',
"IPython>=5.0.0",
"jupyter",
diff --git a/src/pyquickhelper/__main__.py b/src/pyquickhelper/__main__.py
index ecb154617..613253958 100644
--- a/src/pyquickhelper/__main__.py
+++ b/src/pyquickhelper/__main__.py
@@ -31,6 +31,7 @@ def main(args, fLOG=print):
from .cli.notebook import run_notebook, convert_notebook
from .loghelper import set_password
from .filehelper.download_urls_helper import download_urls_in_folder_content
+ from .cli.uvicorn_cli import uvicorn_app
except ImportError: # pragma: no cover
from pyquickhelper.cli.pyq_sync_cli import pyq_sync
from pyquickhelper.cli.encryption_file_cli import encrypt_file, decrypt_file
@@ -47,6 +48,7 @@ def main(args, fLOG=print):
from pyquickhelper.cli.notebook import run_notebook, convert_notebook
from pyquickhelper.loghelper import set_password
from pyquickhelper.filehelper.download_urls_helper import download_urls_in_folder_content
+ from pyquickhelper.cli.uvicorn_cli import uvicorn_app
fcts = dict(synchronize_folder=pyq_sync, encrypt_file=encrypt_file,
decrypt_file=decrypt_file, encrypt=encrypt,
@@ -58,7 +60,8 @@ def main(args, fLOG=print):
zoom_img=zoom_img, images2pdf=images2pdf,
repeat_script=repeat_script,
ftp_upload=ftp_upload, set_password=set_password,
- download_urls_in_folder_content=download_urls_in_folder_content)
+ download_urls_in_folder_content=download_urls_in_folder_content,
+ uvicorn_app=uvicorn_app)
return cli_main_helper(fcts, args=args, fLOG=fLOG)
diff --git a/src/pyquickhelper/cli/uvicorn_cli.py b/src/pyquickhelper/cli/uvicorn_cli.py
new file mode 100644
index 000000000..8da9e4e91
--- /dev/null
+++ b/src/pyquickhelper/cli/uvicorn_cli.py
@@ -0,0 +1,31 @@
+"""
+@file
+@brief Simplified function versions.
+"""
+import os
+
+
+def uvicorn_app(path="dummy_db.db3", pwd="dummy", port=8798, host="127.0.0.1"):
+ """
+ Runs a uvicorn application. It should be used for testing
+ not for production. Use ``host:post/redoc`` or
+ ``host:post/docs`` to get a web page in order to
+ submit files.
+
+ :param path: filename for the databse
+ :param pwd: password
+ :param host: host
+ :param port: port
+
+ .. cmdref::
+ :title: Runs a uvicorn application
+ :cmd: -m pyquickhelper uvicorn_app --help
+
+ Runs a uvicorn application.
+ """
+ from ..server.filestore_fastapi import create_app # pylint: disable=W0611
+ import uvicorn
+ os.environ['PYQUICKHELPER_FASTAPI_PWD'] = pwd
+ os.environ['PYQUICKHELPER_FASTAPI_PATH'] = path
+ uvicorn.run("pyquickhelper.server.filestore_fastapi:create_app",
+ host=host, port=port, log_level="info", factory=True)
diff --git a/src/pyquickhelper/helpgen/default_conf.py b/src/pyquickhelper/helpgen/default_conf.py
index 031a41492..604de57d1 100644
--- a/src/pyquickhelper/helpgen/default_conf.py
+++ b/src/pyquickhelper/helpgen/default_conf.py
@@ -94,6 +94,7 @@ def get_epkg_dictionary():
'django': 'https://www.djangoproject.com/',
'docutils': 'http://docutils.sourceforge.net/',
'dvipng': 'https://ctan.org/pkg/dvipng?lang=en',
+ 'FastAPI': 'https://fastapi.tiangolo.com/',
'format style': 'https://pyformat.info/>`_',
'FTP': 'https://en.wikipedia.org/wiki/File_Transfer_Protocol',
'getsitepackages': 'https://docs.python.org/3/library/site.html#site.getsitepackages',
@@ -188,6 +189,7 @@ def get_epkg_dictionary():
'Python': 'http://www.python.org/',
'python-jenkins': 'http://python-jenkins.readthedocs.org/en/latest/',
'pywin32': 'https://sourceforge.net/projects/pywin32/',
+ 'REST': 'https://en.wikipedia.org/wiki/Representational_state_transfer',
'reveal.js': 'https://github.com/hakimel/reveal.js/releases',
'rst': 'https://en.wikipedia.org/wiki/ReStructuredText',
'RST': 'https://en.wikipedia.org/wiki/ReStructuredText',
@@ -218,6 +220,7 @@ def get_epkg_dictionary():
'tornado': 'http://www.tornadoweb.org/en/stable/',
'TortoiseSVN': 'http://tortoisesvn.net/',
'travis': 'https://travis-ci.org/',
+ 'uvicorn': 'https://www.uvicorn.org/',
'vis.js': 'https://visjs.org/',
'viz.js': 'https://github.com/mdaines/viz.js/',
'Visual Studio Community Edition 2015': 'https://imagine.microsoft.com/en-us/Catalog/Product/101',
diff --git a/src/pyquickhelper/ipythonhelper/magic_class_diff.py b/src/pyquickhelper/ipythonhelper/magic_class_diff.py
index 957cb7136..714bce529 100644
--- a/src/pyquickhelper/ipythonhelper/magic_class_diff.py
+++ b/src/pyquickhelper/ipythonhelper/magic_class_diff.py
@@ -77,7 +77,8 @@ def textdiff(self, line):
if args is not None:
html, js = create_visual_diff_through_html_files(
args.f1, args.f2, encoding=args.encoding, notebook=True,
- context_size=None if args.context in [None, ""] else int(args.context),
+ context_size=None if args.context in [
+ None, ""] else int(args.context),
inline_view=args.inline)
display_html(html)
return js
diff --git a/src/pyquickhelper/server/filestore_fastapi.py b/src/pyquickhelper/server/filestore_fastapi.py
new file mode 100644
index 000000000..1f4fbd190
--- /dev/null
+++ b/src/pyquickhelper/server/filestore_fastapi.py
@@ -0,0 +1,114 @@
+# -*- coding:utf-8 -*-
+"""
+@file
+@brief Simple class to store and retrieve files through an API.
+"""
+import os
+from typing import Optional
+from fastapi import FastAPI, Request, HTTPException
+from pydantic import BaseModel # pylint: disable=E0611
+from .filestore_sqlite import SqlLite3FileStore
+
+
+class Item(BaseModel):
+ name: Optional[str] # pylint: disable=E1136
+ format: Optional[str] # pylint: disable=E1136
+ team: Optional[str] # pylint: disable=E1136
+ project: Optional[str] # pylint: disable=E1136
+ version: Optional[str] # pylint: disable=E1136
+ content: Optional[str] # pylint: disable=E1136
+ password: str
+
+
+class Metric(BaseModel):
+ name: str
+ password: str
+
+
+class Query(BaseModel):
+ name: Optional[str] # pylint: disable=E1136
+ team: Optional[str] # pylint: disable=E1136
+ project: Optional[str] # pylint: disable=E1136
+ version: Optional[str] # pylint: disable=E1136
+ password: str
+
+
+def create_fast_api_app(db_path, password):
+ """
+ Creates a :epkg:`REST` application based on :epkg:`FastAPI`.
+
+ :return: app
+ """
+ store = SqlLite3FileStore(db_path)
+
+ async def get_root():
+ return {"pyquickhelper": "FastAPI to load and query files"}
+
+ async def add(item: Item, request: Request):
+ if item.password != password:
+ raise HTTPException(status_code=401, detail="Wrong password")
+ kwargs = dict(name=item.name, format=item.format,
+ team=item.team, project=item.project,
+ version=item.version, content=item.content)
+ kwargs['metadata'] = dict(client=request.client)
+ res = store.add(**kwargs)
+ if 'content' in res:
+ del res['content']
+ return res
+
+ async def metrics(query: Metric, request: Request):
+ if query.password != password:
+ raise HTTPException(status_code=401, detail="Wrong password")
+ res = list(store.enumerate_data(name=query.name, join=True))
+ return res
+
+ async def query(query: Query, request: Request):
+ if query.password != password:
+ raise HTTPException(status_code=401, detail="Wrong password")
+ res = list(store.enumerate(name=query.name, team=query.team,
+ project=query.project, version=query.version))
+ return res
+
+ app = FastAPI()
+ app.get("/")(get_root)
+ app.post("/add/")(add)
+ app.post("/metrics/")(metrics)
+ app.post("/query/")(query)
+ return app
+
+
+def create_app():
+ """
+ Creates an instance of application class returned
+ by @see fn create_fast_api_app. It checks that
+ environment variables ``PYQUICKHELPER_FASTAPI_PWD``
+ and ``PYQUICKHELPER_FASTAPI_PATH`` are set up with
+ a password and a filename. Otherwise, the function
+ raised an exception.
+
+ Inspired from the guidelines
+ `uvicorn/deployment `_,
+ `(2) `_.
+ Some command lines:
+
+ ::
+
+ uvicorn --factory pyquickhelper.server.filestore_fastapi:create_app --port 8798
+ --ssl-keyfile=./key.pem --ssl-certfile=./cert.pem
+ gunicorn --keyfile=./key.pem --certfile=./cert.pem -k uvicorn.workers.UvicornWorker
+ --factory pyquickhelper.server.filestore_fastapi:create_app
+
+ ::
+
+ uvicorn.run("pyquickhelper.server.filestore_fastapi:create_app",
+ host="127.0.0.1", port=8798, log_level="info", factory=True)
+ """
+ if "PYQUICKHELPER_FASTAPI_PWD" not in os.environ:
+ raise RuntimeError(
+ "Environment variable PYQUICKHELPER_FASTAPI_PWD is missing.")
+ if "PYQUICKHELPER_FASTAPI_PATH" not in os.environ:
+ raise RuntimeError(
+ "Environment variable PYQUICKHELPER_FASTAPI_PATH is missing.")
+ app = create_fast_api_app(os.environ['PYQUICKHELPER_FASTAPI_PATH'],
+ os.environ['PYQUICKHELPER_FASTAPI_PWD'])
+ return app
diff --git a/src/pyquickhelper/server/filestore_sqlite.py b/src/pyquickhelper/server/filestore_sqlite.py
new file mode 100644
index 000000000..f45f1955d
--- /dev/null
+++ b/src/pyquickhelper/server/filestore_sqlite.py
@@ -0,0 +1,237 @@
+# -*- coding:utf-8 -*-
+"""
+@file
+@brief Simple class to store and retrieve files with a sqlite3 detabase.
+"""
+import os
+import io
+import sqlite3
+import json
+from datetime import datetime
+import pandas
+
+
+class SqlLite3FileStore:
+ """
+ Simple file storage implemented with :epkg:`python:sqlite3`.
+
+ :param path: location of the database.
+ """
+
+ def __init__(self, path="_file_store_.db3"):
+ self.path_ = path
+ self._create()
+
+ def _create(self):
+ """
+ Creates the database if it does not exists.
+ """
+ self.con_ = sqlite3.connect(self.path_)
+ cur = self.con_.cursor()
+ cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
+ res = cur.fetchall()
+ commit = False
+ if ('files',) not in res:
+ cur.execute(
+ '''CREATE TABLE files
+ (id INTEGER PRIMARY KEY, date TEXT, name TEXT,
+ format TEXT, metadata TEXT, team TEXT,
+ project TEXT, version TEXT, content BLOB)''')
+ commit = True
+ if ('files',) not in res:
+ cur.execute(
+ '''CREATE TABLE data
+ (id INTEGER PRIMARY KEY, idfile INTEGER,
+ name TEXT, value REAL, date TEXT)''')
+ commit = True
+ if commit:
+ self.con_.commit()
+
+ def add(self, name, content, format=None, date=None, metadata=None,
+ team=None, project=None, version=None):
+ """
+ Adds a file to the database.
+
+ :param name: filename
+ :param content: file content (it can be a dataframe)
+ :param format: format
+ :param date: date, by default now
+ :param metadata: addition information
+ :param team: another name
+ :param project: another name
+ :param version: version
+ :return: added data as a dictionary (no content)
+ """
+ if date is None:
+ date = datetime.now()
+ date = date.isoformat()
+ if isinstance(metadata, dict):
+ metadata = json.dumps(metadata)
+ elif metadata is not None:
+ raise TypeError(
+ "metadata must be None or a dictionary.")
+ if isinstance(content, pandas.DataFrame):
+ st = io.StringIO()
+ content.to_csv(st, index=False, encoding="utf-8")
+ content = st.getvalue()
+ if format is None:
+ format = "df"
+ if format is None:
+ format = os.path.splitext(name)[-1]
+ record = dict(name=name, content=content, format=format,
+ metadata=metadata, team=team, project=project,
+ version=version, date=date)
+ fields = []
+ values = []
+ for k, n in record.items():
+ if n is None:
+ continue
+ fields.append(k)
+ values.append(n.replace("\\", "\\\\").replace("'", "''"))
+ sqlite_insert_blob_query = """
+ INSERT INTO files (%s) VALUES (%s)""" % (
+ ",".join(fields), ",".join("'%s'" % v for v in values))
+ cur = self.con_.cursor()
+ cur.execute(sqlite_insert_blob_query)
+ self.con_.commit()
+ output = dict(name=name, format=format,
+ metadata=metadata, team=team, project=project,
+ version=version, date=date)
+ return {k: v for k, v in output.items() if v is not None}
+
+ def add_data(self, idfile, name, value, date=None):
+ """
+ Adds a file to the database.
+
+ :param idfile: refers to database files
+ :param date: date, by default now
+ :param name: name
+ :param value: data value
+ :return: added data
+ """
+ if date is None:
+ date = datetime.now()
+ date = date.isoformat()
+
+ fields = ['idfile', 'date', 'name', 'value']
+ values = [idfile, date, name, value]
+ sqlite_insert_blob_query = """
+ INSERT INTO data (%s) VALUES (%s)""" % (
+ ",".join(fields), ",".join("%r" % v for v in values))
+ cur = self.con_.cursor()
+ cur.execute(sqlite_insert_blob_query)
+ self.con_.commit()
+
+ def _enumerate(self, condition, fields):
+ cur = self.con_.cursor()
+ query = '''SELECT %s FROM files WHERE %s''' % (
+ ",".join(fields), " AND ".join(condition))
+ res = cur.execute(query)
+
+ for line in res:
+ res = {k: v for k, v in zip(fields, line)} # pylint: disable=R1721
+ if 'format' in res and 'content' in res and res['format'] == 'df':
+ st = io.StringIO(res['content'])
+ df = pandas.read_csv(st, encoding="utf-8")
+ res['content'] = df
+ if 'metadata' in res and res['metadata']:
+ res['metadata'] = json.loads(res['metadata'])
+ yield res
+
+ def enumerate_content(self, name=None, format=None, date=None, metadata=None,
+ team=None, project=None, version=None):
+ """
+ Queries the database, enumerates the results,
+ returns the content as well.
+
+ :param name: filename
+ :param format: format
+ :param date: date, by default now
+ :param metadata: addition information
+ :param team: another name
+ :param project: another name
+ :param version: version
+ :return: results
+ """
+ record = dict(name=name, format=format,
+ metadata=metadata, team=team, project=project,
+ version=version, date=date)
+ cond = []
+ for k, v in record.items():
+ if v is None:
+ continue
+ cond.append('%s="%s"' % (k, v))
+ fields = ["id", "name", "format", "date", "metadata",
+ "team", "project", "version", "content"]
+ for it in self._enumerate(cond, fields):
+ yield it
+
+ def enumerate(self, name=None, format=None, date=None, metadata=None,
+ team=None, project=None, version=None):
+ """
+ Queries the database, enumerates the results.
+
+ :param name: filename
+ :param format: format
+ :param date: date, by default now
+ :param metadata: addition information
+ :param team: another name
+ :param project: another name
+ :param version: version
+ :return: results
+ """
+ record = dict(name=name, format=format,
+ metadata=metadata, team=team, project=project,
+ version=version, date=date)
+ cond = []
+ for k, v in record.items():
+ if v is None:
+ continue
+ cond.append('%s="%s"' % (k, v))
+ fields = ["id", "name", "format", "date", "metadata",
+ "team", "project", "version"]
+ for it in self._enumerate(cond, fields):
+ yield it
+
+ def enumerate_data(self, idfile=None, name=None, join=False):
+ """
+ Queries the database, enumerates the results.
+
+ :param idfile: file identifier
+ :param name: value name, None if not specified
+ :param join: join with the table *files*
+ :return: results
+ """
+ record = dict(name=name, idfile=idfile)
+ cond = []
+ for k, v in record.items():
+ if v is None:
+ continue
+ if join:
+ cond.append('data.%s="%s"' % (k, v))
+ else:
+ cond.append('%s="%s"' % (k, v))
+ cur = self.con_.cursor()
+ if join:
+ fields = ["data.id", "idfile", "data.name", "data.date", "value"]
+ fields2 = ['name', 'project', 'team', 'version']
+ query = '''
+ SELECT %s, %s
+ FROM data INNER JOIN files AS B on B.id = idfile
+ WHERE %s''' % (
+ ",".join(fields),
+ ",".join(map(lambda s: "B.%s" % s, fields2)),
+ " AND ".join(cond))
+ else:
+ fields = ["id", "idfile", "name", "date", "value"]
+ query = '''SELECT %s FROM data WHERE %s''' % (
+ ",".join(fields), " AND ".join(cond))
+ res = cur.execute(query)
+
+ if join:
+ fields = ([s.replace('data.', '') for s in fields] +
+ ['name_f', 'project', 'team', 'version'])
+ for line in res:
+ res = {k: v for k, v in zip(fields, line) # pylint: disable=R1721
+ if v is not None}
+ yield res