diff --git a/.gitignore b/.gitignore index 4817b43b..402f640b 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ celerybeat-schedule # dotenv .env +NIPTool/.env # virtualenv .venv diff --git a/NIPTool/API/external/api/api_v1/api.py b/NIPTool/API/external/api/api_v1/api.py deleted file mode 100644 index a774ba8c..00000000 --- a/NIPTool/API/external/api/api_v1/api.py +++ /dev/null @@ -1,21 +0,0 @@ -from fastapi import FastAPI - -from NIPTool.API.external.api.api_v1.endpoints import ( - batches, - index, - sample, - update, - download, - statistics, - login, -) - -app = FastAPI() - -app.include_router(login.router, prefix="/api/v1/login", tags=["login"]) -app.include_router(batches.router, prefix="/api/v1/batches", tags=["batches"]) -app.include_router(index.router, prefix="/api/v1", tags=["index"]) -app.include_router(sample.router, prefix="/api/v1", tags=["sample"]) -app.include_router(update.router, prefix="/api/v1", tags=["update"]) -app.include_router(download.router, prefix="/api/v1", tags=["download"]) -app.include_router(statistics.router, prefix="/api/v1", tags=["statistics"]) diff --git a/NIPTool/API/external/api/api_v1/endpoints/batches.py b/NIPTool/API/external/api/api_v1/endpoints/batches.py index 4d1716f5..91e30974 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/batches.py +++ b/NIPTool/API/external/api/api_v1/endpoints/batches.py @@ -1,15 +1,12 @@ from fastapi import APIRouter, Depends, Request from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.API.external.constants import TRISOMI_TRESHOLDS +from NIPTool.API.external.utils import * +from NIPTool.config import get_nipt_adapter, templates from NIPTool.crud import find from NIPTool.models.database import Batch, User -from NIPTool.API.external.utils import * -from NIPTool.API.external.constants import TRISOMI_TRESHOLDS -from NIPTool.API.external.api.deps import get_nipt_adapter - -from fastapi.templating import Jinja2Templates router = APIRouter() -templates = Jinja2Templates(directory="templates") CURRENT_USER = User(username="mayapapaya", email="mayabrandi@123.com", role="RW").dict() @@ -18,7 +15,6 @@ def batches( request: Request, adapter: NiptAdapter = Depends(get_nipt_adapter) ): # , user: User = Depends(get_current_active_user)): """List of all batches""" - all_batches: List[Batch] = find.batches(adapter=adapter) return templates.TemplateResponse( "batches.html", @@ -36,8 +32,8 @@ def batches( request: Request, adapter: NiptAdapter = Depends(get_nipt_adapter) ): # , user: User = Depends(get_current_active_user)): """List of all batches""" - all_batches: List[Batch] = find.batches(adapter=adapter) + return templates.TemplateResponse( "batches.html", context={ diff --git a/NIPTool/API/external/api/api_v1/endpoints/download.py b/NIPTool/API/external/api/api_v1/endpoints/download.py index b333d867..a6cb0b03 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/download.py +++ b/NIPTool/API/external/api/api_v1/endpoints/download.py @@ -1,13 +1,13 @@ +from pathlib import Path + from fastapi import APIRouter, Depends, Request +from fastapi.responses import FileResponse, RedirectResponse from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.API.external.api.deps import get_nipt_adapter, get_current_active_user -from fastapi.templating import Jinja2Templates +from NIPTool.config import get_nipt_adapter, templates +from NIPTool.crud import find from NIPTool.parse.batch import validate_file_path -from pathlib import Path -from fastapi.responses import RedirectResponse, FileResponse router = APIRouter() -templates = Jinja2Templates(directory="templates") @router.get("/batch_download/{batch_id}/{file_id}") @@ -16,11 +16,11 @@ def batch_download( ): """View for batch downloads""" - batch = adapter.batch(batch_id) + batch: dict = find.batch(adapter=adapter, batch_id=batch_id).dict() file_path = batch.get(file_id) if not validate_file_path(file_path): - # handle the redirect responce! + # handle the redirect response! return RedirectResponse(request.url) path = Path(file_path) @@ -36,10 +36,10 @@ def sample_download( ): """View for sample downloads""" - sample = adapter.sample(sample_id) + sample: dict = find.sample(adapter=adapter, sample_id=sample_id).dict() file_path = sample.get(file_id) if not validate_file_path(file_path): - # handle the redirect responce! + # handle the redirect response! return RedirectResponse(request.url) file = Path(file_path) diff --git a/NIPTool/API/external/api/api_v1/endpoints/index.py b/NIPTool/API/external/api/api_v1/endpoints/index.py index 4d129bd7..3061fa85 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/index.py +++ b/NIPTool/API/external/api/api_v1/endpoints/index.py @@ -1,12 +1,9 @@ - from fastapi import APIRouter, Request -from fastapi.templating import Jinja2Templates - - -templates = Jinja2Templates(directory="templates") +from NIPTool.config import templates router = APIRouter() + @router.post("/") def index(request: Request): """Log in view.""" diff --git a/NIPTool/API/external/api/api_v1/endpoints/login.py b/NIPTool/API/external/api/api_v1/endpoints/login.py index 83c538ed..b7ac759f 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/login.py +++ b/NIPTool/API/external/api/api_v1/endpoints/login.py @@ -1,17 +1,10 @@ -from fastapi import APIRouter, Response -from fastapi.responses import RedirectResponse +from datetime import timedelta -from fastapi import Depends, HTTPException, status, Security +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.responses import RedirectResponse from fastapi.security import OAuth2PasswordRequestForm -from NIPTool.models.server.login import Token, UserInDB, User - -from NIPTool.API.external.api.deps import ( - get_current_active_user, - authenticate_user, - create_access_token, - temp_get_config, -) -from datetime import timedelta +from NIPTool.API.external.api.deps import authenticate_user, create_access_token, temp_get_config +from NIPTool.models.server.login import Token, UserInDB router = APIRouter() diff --git a/NIPTool/API/external/api/api_v1/endpoints/sample.py b/NIPTool/API/external/api/api_v1/endpoints/sample.py index bfddfd60..8000c4be 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/sample.py +++ b/NIPTool/API/external/api/api_v1/endpoints/sample.py @@ -1,19 +1,16 @@ from fastapi import APIRouter, Depends, Request from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.crud import find -from NIPTool.models.database import User from NIPTool.API.external.utils import * -from NIPTool.API.external.api.deps import get_nipt_adapter - -from fastapi.templating import Jinja2Templates +from NIPTool.config import get_nipt_adapter, templates +from NIPTool.crud import find +from NIPTool.models.database import Batch, User router = APIRouter() -templates = Jinja2Templates(directory="templates") @router.get("/samples/{sample_id}/") def sample(request: Request, sample_id: str, adapter: NiptAdapter = Depends(get_nipt_adapter)): - """Sample view with sample information.""" + """Get sample with id""" sample: dict = find.sample(sample_id=sample_id, adapter=adapter).dict() batch: Batch = find.batch(batch_id=sample.get("batch_id"), adapter=adapter) @@ -34,7 +31,7 @@ def sample(request: Request, sample_id: str, adapter: NiptAdapter = Depends(get_ @router.post("/samples/{sample_id}/") def sample(request: Request, sample_id: str, adapter: NiptAdapter = Depends(get_nipt_adapter)): - """Sample view with sample information.""" + """Post sample with id""" sample: dict = find.sample(sample_id=sample_id, adapter=adapter).dict() batch: Batch = find.batch(batch_id=sample.get("batch_id"), adapter=adapter) diff --git a/NIPTool/API/external/api/api_v1/endpoints/statistics.py b/NIPTool/API/external/api/api_v1/endpoints/statistics.py index 8fece86c..09d5dbc8 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/statistics.py +++ b/NIPTool/API/external/api/api_v1/endpoints/statistics.py @@ -1,19 +1,14 @@ -from typing import List - from fastapi import APIRouter, Depends, Request from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.models.database import User from NIPTool.API.external.utils import ( get_last_batches, get_statistics_for_box_plot, get_statistics_for_scatter_plot, ) -from NIPTool.API.external.api.deps import get_nipt_adapter - -from fastapi.templating import Jinja2Templates +from NIPTool.config import get_nipt_adapter, templates +from NIPTool.models.database import User router = APIRouter() -templates = Jinja2Templates(directory="templates") @router.get("/statistics") diff --git a/NIPTool/API/external/api/api_v1/endpoints/update.py b/NIPTool/API/external/api/api_v1/endpoints/update.py index 220b84e6..7cd51882 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/update.py +++ b/NIPTool/API/external/api/api_v1/endpoints/update.py @@ -1,17 +1,15 @@ import logging +from datetime import datetime from typing import Iterable from fastapi import APIRouter, Depends, Request from fastapi.responses import RedirectResponse -from starlette.datastructures import FormData - from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.API.external.utils import * +from NIPTool.config import get_nipt_adapter from NIPTool.crud import update - from NIPTool.models.database import User -from NIPTool.API.external.utils import * -from NIPTool.API.external.api.deps import get_nipt_adapter, get_current_active_user -from datetime import datetime +from starlette.datastructures import FormData router = APIRouter() diff --git a/NIPTool/API/external/api/deps.py b/NIPTool/API/external/api/deps.py index d6c4b33b..381392ab 100644 --- a/NIPTool/API/external/api/deps.py +++ b/NIPTool/API/external/api/deps.py @@ -1,23 +1,14 @@ -from pydantic import BaseSettings - -from NIPTool.adapter.plugin import NiptAdapter -from pymongo import MongoClient -from fastapi.security import OAuth2PasswordBearer -from fastapi import HTTPException, status -from NIPTool.models.server.login import User, UserInDB, TokenData -from fastapi import Depends -from passlib.context import CryptContext from datetime import datetime, timedelta from typing import Optional -from jose import JWTError, jwt - - -class Settings(BaseSettings): - db_uri: str = "test_uri" - db_name: str = "test_db" - -settings = Settings() +from fastapi import Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer +from jose import JWTError, jwt +from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.config import get_nipt_adapter +from NIPTool.crud import find +from NIPTool.models.server.login import TokenData, User, UserInDB +from passlib.context import CryptContext def temp_get_config(): @@ -30,11 +21,6 @@ def temp_get_config(): } -def get_nipt_adapter(): - client = MongoClient(settings.db_uri) - return NiptAdapter(client, db_name=settings.db_name) - - oauth2_scheme = OAuth2PasswordBearer( tokenUrl="/api/v1/login/token", scopes={"me": "Read information about the current user.", "items": "Read items."}, @@ -61,10 +47,10 @@ def get_current_user( token_data = TokenData(username=username) except JWTError: raise credentials_exception - user = adapter.user(token_data.username) + user: User = find.user(adapter=adapter, user_name=token_data.username) if not user: raise credentials_exception - return User(**user) + return user def get_current_active_user(current_user: User = Depends(get_current_user)) -> User: @@ -81,26 +67,25 @@ def get_password_hash(password): return pwd_context.hash(password) -def authenticate_user(username: str, password: str) -> UserInDB: +def authenticate_user(username: str, password: str) -> Optional[UserInDB]: adapter = get_nipt_adapter() - user_dict = adapter.user(username) - if not user_dict: - return False - user = UserInDB(**user_dict) + user: User = find.user(adapter=adapter, user_name=username) if not user: - return False - if not verify_password(password, user.hashed_password): - return False - return user + return None + db_user = UserInDB(**user.dict()) + if not db_user: + return None + if not verify_password(password, db_user.hashed_password): + return None + return db_user def create_access_token(data: dict, expires_delta: Optional[timedelta] = None): - config: dict = temp_get_config() + configs: dict = temp_get_config() to_encode = data.copy() if expires_delta: expire = datetime.utcnow() + expires_delta else: expire = datetime.utcnow() + timedelta(minutes=15) to_encode.update({"exp": expire}) - encoded_jwt = jwt.encode(to_encode, config.get("SECRET_KEY"), algorithm=config.get("ALGORITHM")) - return encoded_jwt + return jwt.encode(to_encode, configs.get("SECRET_KEY"), algorithm=configs.get("ALGORITHM")) diff --git a/NIPTool/API/internal/api/api_v1/api.py b/NIPTool/API/internal/api/api_v1/api.py deleted file mode 100644 index b60ffe66..00000000 --- a/NIPTool/API/internal/api/api_v1/api.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import FastAPI - -from NIPTool.API.internal.api.api_v1.endpoints import insert - - -app = FastAPI() - -app.include_router(insert.router, prefix="/api/v1/insert", tags=["insert"]) diff --git a/NIPTool/API/internal/api/api_v1/endpoints/insert.py b/NIPTool/API/internal/api/api_v1/endpoints/insert.py index 152da8f7..d7634efb 100644 --- a/NIPTool/API/internal/api/api_v1/endpoints/insert.py +++ b/NIPTool/API/internal/api/api_v1/endpoints/insert.py @@ -3,12 +3,12 @@ from fastapi import APIRouter, Depends, Response, status from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.crud.insert import insert_batch, insert_samples, insert_user +from NIPTool.config import get_nipt_adapter from NIPTool.crud import find +from NIPTool.crud.insert import insert_batch, insert_samples, insert_user from NIPTool.models.database import Batch, Sample from NIPTool.models.server.load import BatchRequestBody, UserRequestBody from NIPTool.parse.batch import get_batch, get_samples -from NIPTool.API.internal.api.deps import get_nipt_adapter router = APIRouter() @@ -20,7 +20,6 @@ def batch( adapter: NiptAdapter = Depends(get_nipt_adapter), ): """Function to load batch data into the database with rest""" - nipt_results = Path(batch_files.result_file) if not nipt_results.exists(): response.status_code = status.HTTP_422_UNPROCESSABLE_ENTITY @@ -28,7 +27,7 @@ def batch( samples: List[Sample] = get_samples(nipt_results) batch: Batch = get_batch(nipt_results) if find.batch(adapter=adapter, batch_id=batch.batch_id): - return "batch allready in database" + return "batch already in database" insert_batch(adapter=adapter, batch=batch, batch_files=batch_files) insert_samples(adapter=adapter, samples=samples, segmental_calls=batch_files.segmental_calls) @@ -44,7 +43,7 @@ def user( """Function to load user into the database with rest""" if find.user(adapter=adapter, email=user.email): - return "user allready in database" + return "user already in database" insert_user(adapter=adapter, user=user) response.status_code = status.HTTP_200_OK diff --git a/NIPTool/API/internal/api/deps.py b/NIPTool/API/internal/api/deps.py deleted file mode 100644 index 25570662..00000000 --- a/NIPTool/API/internal/api/deps.py +++ /dev/null @@ -1,16 +0,0 @@ -from NIPTool.adapter.plugin import NiptAdapter -from pymongo import MongoClient -from pydantic import BaseSettings - - -class Settings(BaseSettings): - db_uri: str = "test_uri" - db_name: str = "test_db" - - -settings = Settings() - - -def get_nipt_adapter(): - client = MongoClient(settings.db_uri) - return NiptAdapter(client, db_name=settings.db_name) diff --git a/NIPTool/commands/base.py b/NIPTool/commands/base.py index a271a6ce..b065eac9 100644 --- a/NIPTool/commands/base.py +++ b/NIPTool/commands/base.py @@ -1,52 +1,51 @@ -#!/usr/bin/env python -import logging +""" +Module with CLI commands for NIPTool -import click +The CLI is intended for development/testing purpose only. To run in a production setting please refer to documentation +for suggestions how. -from flask.cli import FlaskGroup, with_appcontext -from flask import current_app +""" +import logging -# commands -from NIPTool.server import create_app, configure_app +import click +import pkg_resources +import uvicorn # Get version and doc decorator from NIPTool import __version__ +from NIPTool.commands.load_commands import load_commands +from NIPTool.config import settings LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] LOG = logging.getLogger(__name__) +ENV_FILE = pkg_resources.resource_filename("NIPTool", ".env") + @click.version_option(__version__) -@click.group( - cls=FlaskGroup, - create_app=create_app, - add_default_commands=True, - invoke_without_command=False, - add_version_option=False) -@click.option("-c", "--config", type=click.File(), help="Path to config yaml file") -@with_appcontext -def cli(config): +@click.group() +@click.pass_context +def cli(context: click.Context): """ Main entry point """ - if current_app.test: - return - configure_app(current_app, config) - pass - + logging.basicConfig(level=logging.INFO) + context.obj = {} -@cli.command() -def test(): - """Test server using CLI""" - click.echo("test") - pass +@cli.command(name="serve") +@click.option( + "--api", default="external", type=click.Choice(["external", "internal"]), show_default=True +) +@click.option("--reload", is_flag=True) +def serve_command(reload: bool, api: str): + """Serve the NIPT app for testing purpose. -@cli.command() -@with_appcontext -def name(): - """Returns the app name, for testing purposes, mostly""" - click.echo(current_app.name) - return current_app.name + This command will serve the user interface (external) as default + """ + app = "NIPTool.main:external_app" + if api == "internal": + app = "NIPTool.main:internal_app" + LOG.info("Running %s api on host:%s and port:%s", api, settings.host, settings.port) + uvicorn.run(app=app, host=settings.host, port=settings.port, reload=reload) -cli.add_command(test) -cli.add_command(name) +cli.add_command(load_commands) diff --git a/NIPTool/commands/load_commands.py b/NIPTool/commands/load_commands.py new file mode 100644 index 00000000..3508862e --- /dev/null +++ b/NIPTool/commands/load_commands.py @@ -0,0 +1,66 @@ +import logging +from pathlib import Path +from typing import List + +import click +from NIPTool.adapter import NiptAdapter +from NIPTool.config import settings +from NIPTool.crud import find, insert +from NIPTool.exeptions import InsertError +from NIPTool.models.database import Batch, Sample +from NIPTool.models.server.load import BatchRequestBody, UserRequestBody +from NIPTool.parse.batch import get_batch, get_samples +from pymongo import MongoClient + +LOG = logging.getLogger(__name__) + + +@click.group(name="load") +@click.pass_obj +def load_commands(context: dict): + client = MongoClient(settings.db_uri) + context["adapter"] = NiptAdapter(client, db_name=settings.db_name) + LOG.info("Connected to %s", settings.db_name) + + +@load_commands.command(name="batch") +@click.option("--result-file", type=click.Path(exists=True, dir_okay=False), required=True) +@click.option("--multiqc-report", type=click.Path(dir_okay=False)) +@click.option("--segmental-calls", type=click.Path(dir_okay=False)) +@click.pass_obj +def load_batch( + context: dict, result_file: click.Path, multiqc_report: click.Path, segmental_calls: click.Path +): + """Load fluffy result into database""" + batch_files: BatchRequestBody = BatchRequestBody( + result_file=str(result_file), + multiqc_report=str(multiqc_report), + segmental_calls=str(segmental_calls), + ) + adapter: NiptAdapter = context["adapter"] + nipt_results = Path(str(result_file)) + samples: List[Sample] = get_samples(nipt_results) + batch: Batch = get_batch(nipt_results) + if find.batch(adapter=adapter, batch_id=batch.batch_id): + return "batch already in database" + insert.insert_batch(adapter=adapter, batch=batch, batch_files=batch_files) + insert.insert_samples( + adapter=adapter, samples=samples, segmental_calls=batch_files.segmental_calls + ) + + +@load_commands.command(name="user") +@click.option("--email", required=True) +@click.option("--user-name", required=True) +@click.option("--role", type=click.Choice(["RW", "R"]), default="RW", show_default=True) +@click.pass_obj +def load_user(context: dict, email: str, user_name: str, role: str): + """Add a user to the database""" + user: UserRequestBody = UserRequestBody(email=email, username=user_name, role=role) + adapter: NiptAdapter = context["adapter"] + + try: + insert.insert_user(adapter=adapter, user=user) + except InsertError as err: + LOG.warning(err) + raise click.Abort diff --git a/NIPTool/config.py b/NIPTool/config.py new file mode 100644 index 00000000..7233791d --- /dev/null +++ b/NIPTool/config.py @@ -0,0 +1,33 @@ +from pathlib import Path + +from fastapi.templating import Jinja2Templates +from pydantic import BaseSettings +from pymongo import MongoClient + +from NIPTool.adapter.plugin import NiptAdapter + +NIPT_PACKAGE = Path(__file__).parent +PACKAGE_ROOT: Path = NIPT_PACKAGE.parent +ENV_FILE: Path = PACKAGE_ROOT / ".env" +TEMPLATES_DIR: Path = NIPT_PACKAGE / "API" / "external" / "api" / "api_v1" / "templates" + + +class Settings(BaseSettings): + """Settings for serving the nipt app and connect to the mongo database""" + + db_uri: str = "mongodb://localhost:27017/nipt-demo" + db_name: str = "nipt-demo" + host: str = "localhost" + port: int = 8000 + + class Config: + from_file = str(ENV_FILE) + + +settings = Settings() +templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) + + +def get_nipt_adapter(): + client = MongoClient(settings.db_uri) + return NiptAdapter(client, db_name=settings.db_name) diff --git a/NIPTool/crud/find.py b/NIPTool/crud/find.py index 69065879..5da867c7 100644 --- a/NIPTool/crud/find.py +++ b/NIPTool/crud/find.py @@ -1,15 +1,20 @@ from typing import Iterable, List, Optional -from pydantic import parse_obj_as - from NIPTool.adapter import NiptAdapter -from NIPTool.models.database import Batch, User, Sample +from NIPTool.models.database import Batch, Sample, User +from pydantic import parse_obj_as -def user(adapter: NiptAdapter, email: str) -> Optional[User]: +def user( + adapter: NiptAdapter, email: Optional[str] = None, user_name: Optional[str] = None +) -> Optional[User]: """Find user from user collection""" - - raw_user: dict = adapter.user_collection.find_one({"email": email}) # ????? wierd + if email: + raw_user: dict = adapter.user_collection.find_one({"email": email}) # ????? wierd + elif user_name: + raw_user: dict = adapter.user_collection.find_one({"username": user_name}) + else: + raise SyntaxError("Have to use email or user_name") if not raw_user: return None diff --git a/NIPTool/crud/insert.py b/NIPTool/crud/insert.py index 21aed966..c1f44a3d 100644 --- a/NIPTool/crud/insert.py +++ b/NIPTool/crud/insert.py @@ -22,7 +22,7 @@ def insert_batch(adapter: NiptAdapter, batch: Batch, batch_files: BatchRequestBo result: InsertOneResult = adapter.batch_collection.insert_one(batch_dict) LOG.info("Added document %s.", batch_dict["batch_id"]) except: - raise InsertError(message=f"Batch {batch.batch_id} allready in database.") + raise InsertError(message=f"Batch {batch.batch_id} already in database.") return result.inserted_id @@ -41,7 +41,7 @@ def insert_samples( result: InsertManyResult = adapter.sample_collection.insert_many(sample_dicts) LOG.info("Added sample documents.") except: - raise InsertError(f"Sample keys allready in database.") + raise InsertError(f"Sample keys already in database.") return result.inserted_ids @@ -53,9 +53,13 @@ def insert_user(adapter: NiptAdapter, user: UserRequestBody) -> str: "username": user.username, "role": user.role, } + # Check that the username is unique, this should be controlled by creating a index with restrictions + existing_user: Optional[dict] = adapter.user_collection.find_one({"username": user.username}) + if existing_user: + raise InsertError(f"User with username {user.username} already exist in database.") try: result: InsertOneResult = adapter.user_collection.insert_one(user_dict) - LOG.info("Added user documen %s.", user.email) + LOG.info("Added user document %s.", user.email) except: raise InsertError(f"User {user.email} already in database.") return result.inserted_id diff --git a/NIPTool/main.py b/NIPTool/main.py new file mode 100644 index 00000000..5cdd10fb --- /dev/null +++ b/NIPTool/main.py @@ -0,0 +1,24 @@ +from fastapi import FastAPI + +from NIPTool.API.external.api.api_v1.endpoints import ( + batches, + download, + index, + login, + sample, + statistics, + update, +) +from NIPTool.API.internal.api.api_v1.endpoints import insert + +external_app = FastAPI() +external_app.include_router(login.router, prefix="/api/v1/login", tags=["login"]) +external_app.include_router(batches.router, prefix="/api/v1/batches", tags=["batches"]) +external_app.include_router(index.router, prefix="/api/v1", tags=["index"]) +external_app.include_router(sample.router, prefix="/api/v1", tags=["sample"]) +external_app.include_router(update.router, prefix="/api/v1", tags=["update"]) +external_app.include_router(download.router, prefix="/api/v1", tags=["download"]) +external_app.include_router(statistics.router, prefix="/api/v1", tags=["statistics"]) + +internal_app = FastAPI() +internal_app.include_router(insert.router, prefix="/api/v1/insert", tags=["insert"]) diff --git a/NIPTool/models/database/user.py b/NIPTool/models/database/user.py index 1fd927a5..bcb1b553 100644 --- a/NIPTool/models/database/user.py +++ b/NIPTool/models/database/user.py @@ -1,5 +1,6 @@ from pydantic import BaseModel + class User(BaseModel): email: str username: str diff --git a/NIPTool/models/server/load.py b/NIPTool/models/server/load.py index 3b79e4c5..1e020d7c 100644 --- a/NIPTool/models/server/load.py +++ b/NIPTool/models/server/load.py @@ -1,13 +1,16 @@ -from pydantic import BaseModel +from typing import Optional + +from pydantic import BaseModel, EmailStr +from typing_extensions import Literal class BatchRequestBody(BaseModel): result_file: str - multiqc_report: str - segmental_calls: str + multiqc_report: Optional[str] + segmental_calls: Optional[str] class UserRequestBody(BaseModel): - email: str + email: EmailStr username: str - role: str + role: Literal["R", "RW"] diff --git a/NIPTool/tools/__init__.py b/NIPTool/tools/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/NIPTool/tools/cli_utils.py b/NIPTool/tools/cli_utils.py deleted file mode 100644 index 9b8ab14f..00000000 --- a/NIPTool/tools/cli_utils.py +++ /dev/null @@ -1,121 +0,0 @@ -import json -import yaml -import logging -import os -import pathlib -import copy -import collections - -LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] -LOG = logging.getLogger(__name__) - - -def convert_dot(string): - """ - replaces dot with underscore - """ - return string.replace(".", "_") - - -def add_doc(docstring): - """ - A decorator for adding docstring. Taken shamelessly from stackexchange. - """ - - def document(func): - func.__doc__ = docstring - return func - - return document - - -def dict_replace_dot(obj): - """ - recursively replace all dots in json.load keys. - """ - if isinstance(obj, dict): - for key in obj.keys(): - obj[convert_dot(key)] = obj.pop(key) - if isinstance(obj[convert_dot(key)], dict) or isinstance( - obj[convert_dot(key)], list - ): - obj[convert_dot(key)] = dict_replace_dot(obj[convert_dot(key)]) - elif isinstance(obj, list): - for item in obj: - item = dict_replace_dot(item) - return obj - - -def json_read(fname): - """ - Reads JSON file and returns dictionary. Returns error if can't read. - """ - - try: - with open(fname, "r") as f: - analysis_conf = json.load(f, object_hook=dict_replace_dot) - return analysis_conf - except: - LOG.warning("Input config is not JSON") - return False - - -def yaml_read(fname): - """ - Reads YAML file and returns dictionary. Returns error if can't read. - """ - - try: - with open(fname, "r") as f: - analysis_conf = yaml.load(f) - return analysis_conf - except: - LOG.warning("Input config is not YAML") - return False - - -def check_file(fname): - """ - Check file exists and readable. - """ - - path = pathlib.Path(fname) - - if not path.exists() or not path.is_file(): - LOG.error("File not found or input is not a file.") - raise FileNotFoundError - - -def concat_dict_keys(my_dict: dict, key_name="", out_key_list=list()): - """ - Recursively create a list of key:key1,key2 from a nested dictionary - """ - - if isinstance(my_dict, dict): - - if key_name != "": - out_key_list.append(key_name + ":" + ", ".join(list(my_dict.keys()))) - - for k in my_dict.keys(): - concat_dict_keys(my_dict[k], key_name=k, out_key_list=out_key_list) - - return out_key_list - - -def recursive_default_dict(): - """ - Recursivly create defaultdict. - """ - return collections.defaultdict(recursive_default_dict) - - -def convert_defaultdict_to_regular_dict(inputdict: dict): - """ - Recursively convert defaultdict to dict. - """ - if isinstance(inputdict, collections.defaultdict): - inputdict = { - key: convert_defaultdict_to_regular_dict(value) - for key, value in inputdict.items() - } - return inputdict diff --git a/README.md b/README.md index 3a20de3c..cf8b18a5 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ # NIPTool [![Coverage Status](https://coveralls.io/repos/github/Clinical-Genomics/NIPTool/badge.svg?branch=master)](https://coveralls.io/github/Clinical-Genomics/NIPTool?branch=master) [![Build Status](https://travis-ci.org/Clinical-Genomics/NIPTool.svg?branch=master)](https://travis-ci.org/Clinical-Genomics/NIPTool) ![Latest Release](https://img.shields.io/github/v/release/clinical-genomics/NIPTool) -NIPTool is a visualisation tool for NIPT data. +NIPTool is a visualisation tool for the data produced by the [Fluffy] pipeline running [WisecondorX] to analyze NIPT. ## Installation - ```bash -git clone https://github.com/Clinical-Genomics/NIPTool.git +git clone https://github.com/Clinical-Genomics/NIPTool cd NIPTool pip install -r requirements.txt -e . ``` @@ -16,23 +15,29 @@ pip install -r requirements.txt -e . ### Demo -Once installed, you can setup NIPTool by running a few commands using the included command line interface. Given you have a MongoDB server listening on the default port (27017), this is how you would setup a fully working NIPTool demo: +**The CLI is intended for development/testing purpose only. To run in a production setting please refer to documentation +for suggestions how.** + +Once installed, you can set up NIPTool by running a few commands using the included command line interface. +Given you have a MongoDB server listening on the default port (27017), this is how you would set up a fully working +NIPTool demo: ```bash -nipt -c NIPTool/tests/fixtures/nipt_config.yaml load batch -b NIPTool/tests/fixtures/valid_fluffy.csv -nipt -c NIPTool/tests/fixtures/nipt_config.yaml load user -n -r RW -e +nipt load batch --result-file NIPTool/tests/fixtures/valid_fluffy.csv ``` -This will setup an instance of NIPTool with a database called `nipt-demo`. Now run +Settings can be used by exporting the environment variables: `DB_NAME`, `DB_URI`, `HOST`, `PORT` +This will set up an instance of NIPTool with a database called `nipt-demo`. Now run ```bash -nipt run +nipt serve --reload ``` -And play around with the interface. + and play around with the interface. ### Docker image -NIPTool can be runned also as a container. The image is available [on Docker Hub](https://hub.docker.com/repository/docker/clinicalgenomics/niptool) or can be build using the Dockerfile provided in this repository. +NIPTool can also run as a container. The image is available [on Docker Hub][docker-hub] or can be build using the +Dockerfile provided in this repository. To build a new image from the Dockerfile use the commands: `docker build -t niptool .` @@ -40,9 +45,9 @@ To run the image use the following command: `docker run --name niptool niptool n To remove the container, type: `docker rm niptool` - ## Release model -NIPTool development is organised on a flexible Git "Release Flow" branching system. This more or less means that we make releases in release branches which corresponds to stable versions of NIPTool. +NIPTool development is organised on a flexible Git "Release Flow" branching system. This more or less means that we +make releases in release branches which corresponds to stable versions of NIPTool. ### Steps to make a new release: @@ -63,7 +68,8 @@ NIPTool development is organised on a flexible Git "Release Flow" branching syst ### Deploying to production -Use `update-nipt-prod.sh` script to update production both on Hasta and clinical-db. **Please follow the development guide and `servers` repo when doing so. It is also important to keep those involved informed.** +Use `update-nipt-prod.sh` script to update production both on Hasta and clinical-db. +**Please follow the development guide and `servers` repo when doing so. It is also important to keep those involved informed.** ## Back End The NIPT database is a Mongo database consisting of following collections: @@ -72,22 +78,9 @@ The NIPT database is a Mongo database consisting of following collections: - **sample** - holds sample level information. - **user** - holds user names, emails and roles. -The database is loaded through the CLI with data generated by the [FluFFyPipe](https://github.com/Clinical-Genomics/fluffy). - -## CLI -The CLI has two base commands - load and run. The load command is for loading batch and sample data into the nipt database, and the run command is for running the web application. +The database is loaded through the CLI with data generated by the [FluFFyPipe][Fluffy] -### Load - -``` -Usage: nipt -c load batch [OPTIONS] ... -``` - - - -### Run -``` -Usage: nipt run [OPTIONS] ... - -``` +[Fluffy]: https://github.com/Clinical-Genomics/fluffy +[WisecondorX]: https://github.com/CenterForMedicalGeneticsGhent/WisecondorX +[docker-hub]: https://hub.docker.com/repository/docker/clinicalgenomics/niptool \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 1e1c3006..c32e68e0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,5 @@ -flask_debugtoolbar -pytest-flask pytest>=5.2 pytest-cov coveralls pylint -werkzeug<1.0.0 # due to breaking changes in 1.0.0 +mongomock \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f3441acd..dd6c5e16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,18 @@ click bumpversion pymongo -flask mongo_adapter -werkzeug<1.0.0 coloredlogs pyyaml -flask_login -flask_debugtoolbar -authlib -requests -flask_ldap3_login -pandas -pandas-schema + cerberus -fastapi python-multipart +python-dotenv +email-validator +jinja2 +# server stuff +passlib +fastapi +uvicorn +python-jose \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index d5a86ed0..fcd4552b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,15 @@ from pathlib import Path + import pytest +from fastapi.testclient import TestClient from mongomock import MongoClient -from .small_helpers import SmallHelpers from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.API.internal.api.api_v1.api import app -from NIPTool.API.internal.api.deps import get_nipt_adapter -from fastapi.testclient import TestClient +from NIPTool.config import get_nipt_adapter +from NIPTool.main import internal_app as app from NIPTool.models.server.load import BatchRequestBody, UserRequestBody +from .small_helpers import SmallHelpers + DATABASE = "testdb" @@ -16,8 +18,7 @@ def override_nipt_adapter(): mongo_client = MongoClient() database = mongo_client[DATABASE] - adapter = NiptAdapter(database.client, db_name=DATABASE) - return adapter + return NiptAdapter(database.client, db_name=DATABASE) @pytest.fixture() diff --git a/tests/small_helpers.py b/tests/small_helpers.py index a97b9217..c93f5901 100644 --- a/tests/small_helpers.py +++ b/tests/small_helpers.py @@ -3,6 +3,7 @@ class SmallHelpers: """Hold small methods that might be helpful for the tests""" + @staticmethod def batch( batch_id="201860", @@ -28,7 +29,7 @@ def batch( "Stdev_X": 0.029800076293786, "Stdev_Y": 0.0000653186791196846, } - + @staticmethod def sample( batch_id: str = "201860", @@ -113,4 +114,3 @@ def sample( "CNVSegment": "Found", "comment": "None", } -