-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from dockhardman/feature_web_app_init
Feature web app init
- Loading branch information
Showing
27 changed files
with
2,693 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,164 @@ | ||
# Custom | ||
.git | ||
qdrant_storage | ||
|
||
# Byte-compiled / optimized / DLL files | ||
__pycache__/ | ||
*.py[cod] | ||
*$py.class | ||
|
||
# C extensions | ||
*.so | ||
|
||
# Distribution / packaging | ||
.Python | ||
build/ | ||
develop-eggs/ | ||
dist/ | ||
downloads/ | ||
eggs/ | ||
.eggs/ | ||
lib/ | ||
lib64/ | ||
parts/ | ||
sdist/ | ||
var/ | ||
wheels/ | ||
share/python-wheels/ | ||
*.egg-info/ | ||
.installed.cfg | ||
*.egg | ||
MANIFEST | ||
|
||
# PyInstaller | ||
# Usually these files are written by a python script from a template | ||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | ||
*.manifest | ||
*.spec | ||
|
||
# Installer logs | ||
pip-log.txt | ||
pip-delete-this-directory.txt | ||
|
||
# Unit test / coverage reports | ||
htmlcov/ | ||
.tox/ | ||
.nox/ | ||
.coverage | ||
.coverage.* | ||
.cache | ||
nosetests.xml | ||
coverage.xml | ||
*.cover | ||
*.py,cover | ||
.hypothesis/ | ||
.pytest_cache/ | ||
cover/ | ||
|
||
# Translations | ||
*.mo | ||
*.pot | ||
|
||
# Django stuff: | ||
*.log | ||
local_settings.py | ||
db.sqlite3 | ||
db.sqlite3-journal | ||
|
||
# Flask stuff: | ||
instance/ | ||
.webassets-cache | ||
|
||
# Scrapy stuff: | ||
.scrapy | ||
|
||
# Sphinx documentation | ||
docs/_build/ | ||
|
||
# PyBuilder | ||
.pybuilder/ | ||
target/ | ||
|
||
# Jupyter Notebook | ||
.ipynb_checkpoints | ||
|
||
# IPython | ||
profile_default/ | ||
ipython_config.py | ||
|
||
# pyenv | ||
# For a library or package, you might want to ignore these files since the code is | ||
# intended to run in multiple environments; otherwise, check them in: | ||
# .python-version | ||
|
||
# pipenv | ||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. | ||
# However, in case of collaboration, if having platform-specific dependencies or dependencies | ||
# having no cross-platform support, pipenv may install dependencies that don't work, or not | ||
# install all needed dependencies. | ||
#Pipfile.lock | ||
|
||
# poetry | ||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. | ||
# This is especially recommended for binary packages to ensure reproducibility, and is more | ||
# commonly ignored for libraries. | ||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control | ||
#poetry.lock | ||
|
||
# pdm | ||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. | ||
#pdm.lock | ||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it | ||
# in version control. | ||
# https://pdm.fming.dev/#use-with-ide | ||
.pdm.toml | ||
|
||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm | ||
__pypackages__/ | ||
|
||
# Celery stuff | ||
celerybeat-schedule | ||
celerybeat.pid | ||
|
||
# SageMath parsed files | ||
*.sage.py | ||
|
||
# Environments | ||
.env | ||
.venv | ||
env/ | ||
venv/ | ||
ENV/ | ||
env.bak/ | ||
venv.bak/ | ||
|
||
# Spyder project settings | ||
.spyderproject | ||
.spyproject | ||
|
||
# Rope project settings | ||
.ropeproject | ||
|
||
# mkdocs documentation | ||
/site | ||
|
||
# mypy | ||
.mypy_cache/ | ||
.dmypy.json | ||
dmypy.json | ||
|
||
# Pyre type checker | ||
.pyre/ | ||
|
||
# pytype static type analyzer | ||
.pytype/ | ||
|
||
# Cython debug symbols | ||
cython_debug/ | ||
|
||
# PyCharm | ||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can | ||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore | ||
# and can be added to the global gitignore or merged into this file. For a more nuclear | ||
# option (not recommended) you can uncomment the following to ignore the entire idea folder. | ||
#.idea/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
FROM python:3.8-slim | ||
|
||
LABEL maintainer="dockhardman <f1470891079@gmail.com>" | ||
|
||
# Install Dependencies | ||
WORKDIR / | ||
COPY ./requirements.txt ./requirements.txt | ||
RUN pip install -r /requirements.txt | ||
|
||
# Application | ||
WORKDIR /app/ | ||
COPY ./app /app | ||
|
||
CMD ["bash", "/app/start.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
# Developing | ||
logs: | ||
docker logs -f --tail 300 wiki-retrieval-service | ||
|
||
install_all: | ||
poetry install --with dev --no-root | ||
|
||
update_dependencies: | ||
poetry update | ||
poetry export --without-hashes -f requirements.txt --output requirements.txt | ||
poetry export --without-hashes -f requirements.txt --with dev --output requirements-dev.txt | ||
|
||
format: | ||
poetry run black . |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
import os | ||
import copy | ||
import logging | ||
from typing import Text | ||
|
||
from rich.console import Console | ||
|
||
|
||
console = Console() | ||
|
||
|
||
class Settings: | ||
def __init__(self, **kwargs): | ||
environ = copy.deepcopy(os.environ) | ||
environ.update(**kwargs) | ||
|
||
# General | ||
self.APP_NAME: Text = environ.get("APP_NAME", "wiki-retrieval-service") | ||
self.APP_VERSION: Text = environ.get("APP_VERSION", "0.1.0") | ||
self.APP_TIMEZONE: Text = environ.get("APP_TIMEZONE", "Asia/Taipei") | ||
|
||
# Logging Config | ||
self.APP_LOGGER_NAME: Text = environ.get("APP_LOGGER_NAME", "sanic.root") | ||
self.LOG_DIR: Text = "log" | ||
self.LOG_ACCESS_FILENAME: Text = environ.get( | ||
"LOG_ACCESS_FILENAME", "access.log" | ||
) | ||
self.LOG_ERROR_FILENAME: Text = environ.get("LOG_ERROR_FILENAME", "error.log") | ||
self.LOG_SERVICE_FILENAME: Text = environ.get( | ||
"LOG_SERVICE_FILENAME", "service.log" | ||
) | ||
|
||
# OpenAI Config | ||
self.OPENAI_API_KEY = environ.get("OPENAI_API_KEY") | ||
|
||
# Retrieval Config | ||
self.VECTOR_SIZE = int(environ.get("VECTOR_SIZE", "1536")) | ||
self.DATASTORE = environ.get("DATASTORE", "qdrant") | ||
self.BEARER_TOKEN = environ.get("BEARER_TOKEN") | ||
self.QDRANT_URL = environ.get("QDRANT_URL", "wiki-qdrant-service") | ||
self.QDRANT_PORT = int(environ.get("QDRANT_PORT", "6333")) | ||
self.QDRANT_GRPC_PORT = int(environ.get("QDRANT_GRPC_PORT", "6334")) | ||
self.QDRANT_API_KEY = environ.get("QDRANT_API_KEY") | ||
self.QDRANT_COLLECTION = environ.get("QDRANT_COLLECTION", "wiki_documents") | ||
|
||
|
||
settings = Settings() | ||
|
||
logger = logging.getLogger(settings.APP_LOGGER_NAME) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
from .document_store import get_document_store | ||
from .timer import click_timer | ||
|
||
|
||
__all__ = ["click_timer", "get_document_store"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
from sanic.request import Request | ||
|
||
from app.document_store import QdrantDocumentStore | ||
|
||
|
||
def get_document_store(request: Request) -> "QdrantDocumentStore": | ||
return request.app.ctx.document_store |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
from pyassorted.datetime import Timer | ||
from sanic.request import Request | ||
|
||
|
||
def click_timer(request: Request) -> "Timer": | ||
timer = Timer() | ||
timer.click() | ||
return timer |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
from .abc import DocumentStore | ||
from .qdrant import QdrantDocumentStore | ||
|
||
|
||
__all__ = ["DocumentStore", "QdrantDocumentStore"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
from abc import ABC, abstractmethod | ||
from typing import Dict, List, Optional, Text | ||
|
||
from app.schema.models import ( | ||
Document, | ||
DocumentChunk, | ||
DocumentMetadataFilter, | ||
Query, | ||
QueryResult, | ||
QueryWithEmbedding, | ||
) | ||
|
||
|
||
class DocumentStore(ABC): | ||
@property | ||
def host(self) -> str: | ||
raise NotImplementedError | ||
|
||
@property | ||
def port(self) -> int: | ||
raise NotImplementedError | ||
|
||
@abstractmethod | ||
async def touch(self) -> bool: | ||
raise NotImplementedError | ||
|
||
@abstractmethod | ||
async def upsert( | ||
self, documents: List[Document], chunk_token_size: Optional[int] = None | ||
) -> List[Text]: | ||
raise NotImplementedError | ||
|
||
@abstractmethod | ||
async def _upsert(self, chunks: Dict[str, List[DocumentChunk]]) -> List[Text]: | ||
raise NotImplementedError | ||
|
||
@abstractmethod | ||
async def query(self, queries: List[Query]) -> List[QueryResult]: | ||
raise NotImplementedError | ||
|
||
@abstractmethod | ||
async def _query(self, queries: List[QueryWithEmbedding]) -> List[QueryResult]: | ||
raise NotImplementedError | ||
|
||
@abstractmethod | ||
async def delete( | ||
self, | ||
ids: Optional[List[Text]] = None, | ||
filter: Optional[DocumentMetadataFilter] = None, | ||
delete_all: Optional[bool] = None, | ||
) -> bool: | ||
raise NotImplementedError |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
from typing import Text | ||
|
||
from .qdrant import QdrantDocumentStore | ||
|
||
|
||
def get_document_store(collection_name: Text) -> "QdrantDocumentStore": | ||
return QdrantDocumentStore(collection_name=collection_name) |
Oops, something went wrong.