Skip to content
This repository was archived by the owner on Jan 13, 2024. It is now read-only.

Commit cfdf92f

Browse files
authored
Adds class to store files with sqlite3, access with FastAPI (#335)
* Add file store, example with FastAPI * add more functionalities * add query, metric
1 parent eaacf04 commit cfdf92f

File tree

10 files changed

+524
-2
lines changed

10 files changed

+524
-2
lines changed
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
"""
2+
@brief test log(time=1s)
3+
4+
"""
5+
import unittest
6+
import os
7+
import pandas
8+
from pyquickhelper.pycode import ExtTestCase, get_temp_folder
9+
from pyquickhelper.server.filestore_sqlite import SqlLite3FileStore
10+
11+
12+
class TestfileStore(ExtTestCase):
13+
14+
def test_file_store(self):
15+
temp = get_temp_folder(__file__, "temp_file_storage")
16+
name = os.path.join(temp, "filestore.db3")
17+
store = SqlLite3FileStore(name)
18+
df = pandas.DataFrame({"A": ["un", "deux"], "B": [0.5, 0.6]})
19+
store.add(name="zoo", metadata={'hh': 'kk'}, content=df)
20+
got = list(store.enumerate_content(name="zoo"))
21+
self.assertEqual(len(got), 1)
22+
record = got[0]
23+
name = record['name']
24+
self.assertEqual(name, "zoo")
25+
self.assertIn("date", record)
26+
content = record['content']
27+
self.assertIsInstance(df, pandas.DataFrame)
28+
self.assertEqualDataFrame(df, content)
29+
meta = record['metadata']
30+
self.assertIsInstance(meta, dict)
31+
self.assertEqual(meta, {'hh': 'kk'})
32+
got = list(store.enumerate(name="zoo"))
33+
self.assertEqual(len(got), 1)
34+
35+
# data
36+
idfile = record['id']
37+
store.add_data(idfile=idfile, name="ZOO", value="5.6")
38+
res = list(store.enumerate_data(idfile))
39+
self.assertEqual(len(res), 1)
40+
del res[0]['date']
41+
self.assertEqual(res, [{'id': 1, 'idfile': 1, 'name': 'ZOO',
42+
'value': 5.6}])
43+
44+
# data join
45+
res = list(store.enumerate_data(idfile, join=True))
46+
self.assertEqual(len(res), 1)
47+
del res[0]['date']
48+
self.assertEqual(
49+
res, [{'id': 1, 'idfile': 1, 'name': 'ZOO',
50+
'name_f': 'zoo', 'value': 5.6}])
51+
52+
def test_file_store_exc(self):
53+
temp = get_temp_folder(__file__, "temp_file_storage_exc")
54+
name = os.path.join(temp, "filestore.db3")
55+
store = SqlLite3FileStore(name)
56+
df = pandas.DataFrame({"A": ["un", "deux"], "B": [0.5, 0.6]})
57+
self.assertRaise(
58+
lambda: store.add(name="zoo", metadata="{'hh': 'kk'}",
59+
content=df),
60+
TypeError)
61+
62+
def test_file_store1(self):
63+
temp = get_temp_folder(__file__, "temp_file_storage")
64+
name = os.path.join(temp, "filestore.db3")
65+
SqlLite3FileStore(name)
66+
SqlLite3FileStore(name)
67+
68+
69+
if __name__ == "__main__":
70+
unittest.main()
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
"""
2+
@brief test log(time=1s)
3+
4+
"""
5+
import unittest
6+
import os
7+
import pandas
8+
from pyquickhelper.pycode import ExtTestCase, get_temp_folder
9+
from pyquickhelper.server.filestore_fastapi import create_fast_api_app
10+
from fastapi.testclient import TestClient # pylint: disable=E0401
11+
12+
13+
class TestfileStoreRest(ExtTestCase):
14+
15+
def test_file_store(self):
16+
temp = get_temp_folder(__file__, "temp_file_storage_rest")
17+
name = os.path.join(temp, "filestore.db3")
18+
app = create_fast_api_app(name, "BBB")
19+
client = TestClient(app)
20+
response = client.get("/")
21+
self.assertEqual(response.status_code, 200)
22+
self.assertEqual(
23+
response.json(),
24+
{'pyquickhelper': 'FastAPI to load and query files'})
25+
response = client.post(
26+
"/add/", json=dict(name="essai", content="a,b\ne,0\nhh, 1.5",
27+
password="CCC"))
28+
self.assertEqual(response.status_code, 401)
29+
30+
response = client.post(
31+
"/add/", json=dict(name="essai", content="a,b\ne,0\nhh, 1.5",
32+
password="BBB"))
33+
self.assertEqual(response.status_code, 200)
34+
js = response.json()
35+
self.assertIn('date', js)
36+
self.assertNotIn('content', js)
37+
38+
response = client.post(
39+
"/query/", json=dict(name="essai", password="CCC"))
40+
self.assertEqual(response.status_code, 401)
41+
42+
response = client.post(
43+
"/query/", json=dict(name="essai", password="BBB"))
44+
self.assertEqual(response.status_code, 200)
45+
js = response.json()
46+
self.assertEqual(len(js), 1)
47+
48+
response = client.post(
49+
"/metrics/", json=dict(name="essai", password="CCC"))
50+
self.assertEqual(response.status_code, 401)
51+
52+
response = client.post(
53+
"/metrics/", json=dict(name="essai", password="BBB"))
54+
self.assertEqual(response.status_code, 200)
55+
js = response.json()
56+
self.assertEqual(len(js), 0)
57+
58+
59+
if __name__ == "__main__":
60+
unittest.main()

requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ codecov
66
coverage>=5.0
77
cryptography
88
docformatter
9+
fastapi
910
fire
1011
git-pandas
1112
gitdb

setup.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ def write_version():
266266
'keyrings.cryptfile'],
267267
'jenkinshelper': ['python-jenkins>=1.0.0', 'pyyaml'],
268268
'loghelper': ['psutil'],
269+
'server': ['fastapi'],
269270
'all': [
270271
"autopep8", # part of the minimal list
271272
'cffi',
@@ -274,6 +275,7 @@ def write_version():
274275
"docformatter",
275276
"docutils",
276277
'flake8',
278+
'fastapi',
277279
'fire',
278280
"IPython>=5.0.0",
279281
"jupyter",

src/pyquickhelper/__main__.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def main(args, fLOG=print):
3131
from .cli.notebook import run_notebook, convert_notebook
3232
from .loghelper import set_password
3333
from .filehelper.download_urls_helper import download_urls_in_folder_content
34+
from .cli.uvicorn_cli import uvicorn_app
3435
except ImportError: # pragma: no cover
3536
from pyquickhelper.cli.pyq_sync_cli import pyq_sync
3637
from pyquickhelper.cli.encryption_file_cli import encrypt_file, decrypt_file
@@ -47,6 +48,7 @@ def main(args, fLOG=print):
4748
from pyquickhelper.cli.notebook import run_notebook, convert_notebook
4849
from pyquickhelper.loghelper import set_password
4950
from pyquickhelper.filehelper.download_urls_helper import download_urls_in_folder_content
51+
from pyquickhelper.cli.uvicorn_cli import uvicorn_app
5052

5153
fcts = dict(synchronize_folder=pyq_sync, encrypt_file=encrypt_file,
5254
decrypt_file=decrypt_file, encrypt=encrypt,
@@ -58,7 +60,8 @@ def main(args, fLOG=print):
5860
zoom_img=zoom_img, images2pdf=images2pdf,
5961
repeat_script=repeat_script,
6062
ftp_upload=ftp_upload, set_password=set_password,
61-
download_urls_in_folder_content=download_urls_in_folder_content)
63+
download_urls_in_folder_content=download_urls_in_folder_content,
64+
uvicorn_app=uvicorn_app)
6265
return cli_main_helper(fcts, args=args, fLOG=fLOG)
6366

6467

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
@file
3+
@brief Simplified function versions.
4+
"""
5+
import os
6+
7+
8+
def uvicorn_app(path="dummy_db.db3", pwd="dummy", port=8798, host="127.0.0.1"):
9+
"""
10+
Runs a uvicorn application. It should be used for testing
11+
not for production. Use ``host:post/redoc`` or
12+
``host:post/docs`` to get a web page in order to
13+
submit files.
14+
15+
:param path: filename for the databse
16+
:param pwd: password
17+
:param host: host
18+
:param port: port
19+
20+
.. cmdref::
21+
:title: Runs a uvicorn application
22+
:cmd: -m pyquickhelper uvicorn_app --help
23+
24+
Runs a uvicorn application.
25+
"""
26+
from ..server.filestore_fastapi import create_app # pylint: disable=W0611
27+
import uvicorn
28+
os.environ['PYQUICKHELPER_FASTAPI_PWD'] = pwd
29+
os.environ['PYQUICKHELPER_FASTAPI_PATH'] = path
30+
uvicorn.run("pyquickhelper.server.filestore_fastapi:create_app",
31+
host=host, port=port, log_level="info", factory=True)

src/pyquickhelper/helpgen/default_conf.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ def get_epkg_dictionary():
9494
'django': 'https://www.djangoproject.com/',
9595
'docutils': 'http://docutils.sourceforge.net/',
9696
'dvipng': 'https://ctan.org/pkg/dvipng?lang=en',
97+
'FastAPI': 'https://fastapi.tiangolo.com/',
9798
'format style': 'https://pyformat.info/>`_',
9899
'FTP': 'https://en.wikipedia.org/wiki/File_Transfer_Protocol',
99100
'getsitepackages': 'https://docs.python.org/3/library/site.html#site.getsitepackages',
@@ -188,6 +189,7 @@ def get_epkg_dictionary():
188189
'Python': 'http://www.python.org/',
189190
'python-jenkins': 'http://python-jenkins.readthedocs.org/en/latest/',
190191
'pywin32': 'https://sourceforge.net/projects/pywin32/',
192+
'REST': 'https://en.wikipedia.org/wiki/Representational_state_transfer',
191193
'reveal.js': 'https://github.com/hakimel/reveal.js/releases',
192194
'rst': 'https://en.wikipedia.org/wiki/ReStructuredText',
193195
'RST': 'https://en.wikipedia.org/wiki/ReStructuredText',
@@ -218,6 +220,7 @@ def get_epkg_dictionary():
218220
'tornado': 'http://www.tornadoweb.org/en/stable/',
219221
'TortoiseSVN': 'http://tortoisesvn.net/',
220222
'travis': 'https://travis-ci.org/',
223+
'uvicorn': 'https://www.uvicorn.org/',
221224
'vis.js': 'https://visjs.org/',
222225
'viz.js': 'https://github.com/mdaines/viz.js/',
223226
'Visual Studio Community Edition 2015': 'https://imagine.microsoft.com/en-us/Catalog/Product/101',

src/pyquickhelper/ipythonhelper/magic_class_diff.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,8 @@ def textdiff(self, line):
7777
if args is not None:
7878
html, js = create_visual_diff_through_html_files(
7979
args.f1, args.f2, encoding=args.encoding, notebook=True,
80-
context_size=None if args.context in [None, ""] else int(args.context),
80+
context_size=None if args.context in [
81+
None, ""] else int(args.context),
8182
inline_view=args.inline)
8283
display_html(html)
8384
return js
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# -*- coding:utf-8 -*-
2+
"""
3+
@file
4+
@brief Simple class to store and retrieve files through an API.
5+
"""
6+
import os
7+
from typing import Optional
8+
from fastapi import FastAPI, Request, HTTPException
9+
from pydantic import BaseModel # pylint: disable=E0611
10+
from .filestore_sqlite import SqlLite3FileStore
11+
12+
13+
class Item(BaseModel):
14+
name: Optional[str] # pylint: disable=E1136
15+
format: Optional[str] # pylint: disable=E1136
16+
team: Optional[str] # pylint: disable=E1136
17+
project: Optional[str] # pylint: disable=E1136
18+
version: Optional[str] # pylint: disable=E1136
19+
content: Optional[str] # pylint: disable=E1136
20+
password: str
21+
22+
23+
class Metric(BaseModel):
24+
name: str
25+
password: str
26+
27+
28+
class Query(BaseModel):
29+
name: Optional[str] # pylint: disable=E1136
30+
team: Optional[str] # pylint: disable=E1136
31+
project: Optional[str] # pylint: disable=E1136
32+
version: Optional[str] # pylint: disable=E1136
33+
password: str
34+
35+
36+
def create_fast_api_app(db_path, password):
37+
"""
38+
Creates a :epkg:`REST` application based on :epkg:`FastAPI`.
39+
40+
:return: app
41+
"""
42+
store = SqlLite3FileStore(db_path)
43+
44+
async def get_root():
45+
return {"pyquickhelper": "FastAPI to load and query files"}
46+
47+
async def add(item: Item, request: Request):
48+
if item.password != password:
49+
raise HTTPException(status_code=401, detail="Wrong password")
50+
kwargs = dict(name=item.name, format=item.format,
51+
team=item.team, project=item.project,
52+
version=item.version, content=item.content)
53+
kwargs['metadata'] = dict(client=request.client)
54+
res = store.add(**kwargs)
55+
if 'content' in res:
56+
del res['content']
57+
return res
58+
59+
async def metrics(query: Metric, request: Request):
60+
if query.password != password:
61+
raise HTTPException(status_code=401, detail="Wrong password")
62+
res = list(store.enumerate_data(name=query.name, join=True))
63+
return res
64+
65+
async def query(query: Query, request: Request):
66+
if query.password != password:
67+
raise HTTPException(status_code=401, detail="Wrong password")
68+
res = list(store.enumerate(name=query.name, team=query.team,
69+
project=query.project, version=query.version))
70+
return res
71+
72+
app = FastAPI()
73+
app.get("/")(get_root)
74+
app.post("/add/")(add)
75+
app.post("/metrics/")(metrics)
76+
app.post("/query/")(query)
77+
return app
78+
79+
80+
def create_app():
81+
"""
82+
Creates an instance of application class returned
83+
by @see fn create_fast_api_app. It checks that
84+
environment variables ``PYQUICKHELPER_FASTAPI_PWD``
85+
and ``PYQUICKHELPER_FASTAPI_PATH`` are set up with
86+
a password and a filename. Otherwise, the function
87+
raised an exception.
88+
89+
Inspired from the guidelines
90+
`uvicorn/deployment <https://www.uvicorn.org/deployment/>`_,
91+
`(2) <https://www.uvicorn.org/deployment/#running-programmatically>`_.
92+
Some command lines:
93+
94+
::
95+
96+
uvicorn --factory pyquickhelper.server.filestore_fastapi:create_app --port 8798
97+
--ssl-keyfile=./key.pem --ssl-certfile=./cert.pem
98+
gunicorn --keyfile=./key.pem --certfile=./cert.pem -k uvicorn.workers.UvicornWorker
99+
--factory pyquickhelper.server.filestore_fastapi:create_app
100+
101+
::
102+
103+
uvicorn.run("pyquickhelper.server.filestore_fastapi:create_app",
104+
host="127.0.0.1", port=8798, log_level="info", factory=True)
105+
"""
106+
if "PYQUICKHELPER_FASTAPI_PWD" not in os.environ:
107+
raise RuntimeError(
108+
"Environment variable PYQUICKHELPER_FASTAPI_PWD is missing.")
109+
if "PYQUICKHELPER_FASTAPI_PATH" not in os.environ:
110+
raise RuntimeError(
111+
"Environment variable PYQUICKHELPER_FASTAPI_PATH is missing.")
112+
app = create_fast_api_app(os.environ['PYQUICKHELPER_FASTAPI_PATH'],
113+
os.environ['PYQUICKHELPER_FASTAPI_PWD'])
114+
return app

0 commit comments

Comments
 (0)