From 1c697845dd4e40575bb66a0e587d0c8edfb4ed85 Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 10:31:19 +0200 Subject: [PATCH 1/9] Updates dependencies --- .../external/api/api_v1/endpoints/index.py | 2 +- .../internal/api/api_v1/endpoints/insert.py | 6 +- NIPTool/commands/base.py | 82 ++++++++++--------- NIPTool/main.py | 1 + NIPTool/models/database/user.py | 1 + NIPTool/models/server/load.py | 6 +- requirements.txt | 16 ++-- tests/small_helpers.py | 4 +- 8 files changed, 62 insertions(+), 56 deletions(-) create mode 100644 NIPTool/main.py diff --git a/NIPTool/API/external/api/api_v1/endpoints/index.py b/NIPTool/API/external/api/api_v1/endpoints/index.py index 4d129bd7..def32205 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/index.py +++ b/NIPTool/API/external/api/api_v1/endpoints/index.py @@ -1,4 +1,3 @@ - from fastapi import APIRouter, Request from fastapi.templating import Jinja2Templates @@ -7,6 +6,7 @@ router = APIRouter() + @router.post("/") def index(request: Request): """Log in view.""" diff --git a/NIPTool/API/internal/api/api_v1/endpoints/insert.py b/NIPTool/API/internal/api/api_v1/endpoints/insert.py index 152da8f7..b236489c 100644 --- a/NIPTool/API/internal/api/api_v1/endpoints/insert.py +++ b/NIPTool/API/internal/api/api_v1/endpoints/insert.py @@ -3,12 +3,12 @@ from fastapi import APIRouter, Depends, Response, status from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.crud.insert import insert_batch, insert_samples, insert_user +from NIPTool.API.internal.api.deps import get_nipt_adapter from NIPTool.crud import find +from NIPTool.crud.insert import insert_batch, insert_samples, insert_user from NIPTool.models.database import Batch, Sample from NIPTool.models.server.load import BatchRequestBody, UserRequestBody from NIPTool.parse.batch import get_batch, get_samples -from NIPTool.API.internal.api.deps import get_nipt_adapter router = APIRouter() @@ -28,7 +28,7 @@ def batch( samples: List[Sample] = get_samples(nipt_results) batch: Batch = get_batch(nipt_results) if find.batch(adapter=adapter, batch_id=batch.batch_id): - return "batch allready in database" + return "batch already in database" insert_batch(adapter=adapter, batch=batch, batch_files=batch_files) insert_samples(adapter=adapter, samples=samples, segmental_calls=batch_files.segmental_calls) diff --git a/NIPTool/commands/base.py b/NIPTool/commands/base.py index a271a6ce..6ccfe330 100644 --- a/NIPTool/commands/base.py +++ b/NIPTool/commands/base.py @@ -1,52 +1,58 @@ #!/usr/bin/env python import logging +from pathlib import Path +from typing import List import click - -from flask.cli import FlaskGroup, with_appcontext -from flask import current_app - -# commands -from NIPTool.server import create_app, configure_app +from dotenv import dotenv_values, load_dotenv # Get version and doc decorator from NIPTool import __version__ +from NIPTool.adapter import NiptAdapter +from NIPTool.crud import find +from NIPTool.crud.insert import insert_batch, insert_samples +from NIPTool.models.database import Batch, Sample +from NIPTool.models.server.load import BatchRequestBody +from NIPTool.parse.batch import get_batch, get_samples +from pymongo import MongoClient LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] LOG = logging.getLogger(__name__) @click.version_option(__version__) -@click.group( - cls=FlaskGroup, - create_app=create_app, - add_default_commands=True, - invoke_without_command=False, - add_version_option=False) -@click.option("-c", "--config", type=click.File(), help="Path to config yaml file") -@with_appcontext -def cli(config): +@click.group() +@click.pass_context +def cli(context: click.Context): """ Main entry point """ - if current_app.test: - return - configure_app(current_app, config) - pass - - -@cli.command() -def test(): - """Test server using CLI""" - click.echo("test") - pass - - -@cli.command() -@with_appcontext -def name(): - """Returns the app name, for testing purposes, mostly""" - click.echo(current_app.name) - return current_app.name - - -cli.add_command(test) -cli.add_command(name) + logging.basicConfig(level=logging.INFO) + load_dotenv() + settings: dict = dotenv_values(".env") + client = MongoClient(settings["DB_URI"]) + context.obj = {"adapter": NiptAdapter(client, db_name=settings["DB_NAME"])} + LOG.info("Connected to %s", settings["DB_NAME"]) + + +@cli.command(name="load") +@click.option("--result-file", type=click.Path(exists=True, dir_okay=False), required=True) +@click.option("--multiqc-report", type=click.Path(dir_okay=False)) +@click.option("--segmental-calls", type=click.Path(dir_okay=False)) +@click.pass_obj +def load_command( + context: dict, result_file: click.Path, multiqc_report: click.Path, segmental_calls: click.Path +): + """Load fluffy result into database""" + batch_files: BatchRequestBody = BatchRequestBody( + result_file=str(result_file), + multiqc_report=str(multiqc_report), + segmental_calls=str(segmental_calls), + ) + + nipt_results = Path(str(result_file)) + adapter: NiptAdapter = context["adapter"] + samples: List[Sample] = get_samples(nipt_results) + batch: Batch = get_batch(nipt_results) + if find.batch(adapter=adapter, batch_id=batch.batch_id): + return "batch already in database" + insert_batch(adapter=adapter, batch=batch, batch_files=batch_files) + insert_samples(adapter=adapter, samples=samples, segmental_calls=batch_files.segmental_calls) diff --git a/NIPTool/main.py b/NIPTool/main.py new file mode 100644 index 00000000..e331b8ef --- /dev/null +++ b/NIPTool/main.py @@ -0,0 +1 @@ +from NIPTool.API.external.api.api_v1.api import app diff --git a/NIPTool/models/database/user.py b/NIPTool/models/database/user.py index 1fd927a5..bcb1b553 100644 --- a/NIPTool/models/database/user.py +++ b/NIPTool/models/database/user.py @@ -1,5 +1,6 @@ from pydantic import BaseModel + class User(BaseModel): email: str username: str diff --git a/NIPTool/models/server/load.py b/NIPTool/models/server/load.py index 3b79e4c5..7544baed 100644 --- a/NIPTool/models/server/load.py +++ b/NIPTool/models/server/load.py @@ -1,10 +1,12 @@ +from typing import Optional + from pydantic import BaseModel class BatchRequestBody(BaseModel): result_file: str - multiqc_report: str - segmental_calls: str + multiqc_report: Optional[str] + segmental_calls: Optional[str] class UserRequestBody(BaseModel): diff --git a/requirements.txt b/requirements.txt index f3441acd..4610c1f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,15 @@ click bumpversion pymongo -flask mongo_adapter -werkzeug<1.0.0 coloredlogs pyyaml -flask_login -flask_debugtoolbar -authlib -requests -flask_ldap3_login -pandas -pandas-schema + cerberus -fastapi python-multipart +# server stuff +passlib +fastapi +uvicorn +python-jose \ No newline at end of file diff --git a/tests/small_helpers.py b/tests/small_helpers.py index a97b9217..c93f5901 100644 --- a/tests/small_helpers.py +++ b/tests/small_helpers.py @@ -3,6 +3,7 @@ class SmallHelpers: """Hold small methods that might be helpful for the tests""" + @staticmethod def batch( batch_id="201860", @@ -28,7 +29,7 @@ def batch( "Stdev_X": 0.029800076293786, "Stdev_Y": 0.0000653186791196846, } - + @staticmethod def sample( batch_id: str = "201860", @@ -113,4 +114,3 @@ def sample( "CNVSegment": "Found", "comment": "None", } - From b56fdaedb7fabc499f0c836260b574756f32460c Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 15:07:14 +0200 Subject: [PATCH 2/9] Use common dependency to get nipt adapter --- .gitignore | 1 + .../external/api/api_v1/endpoints/batches.py | 13 ++--- .../external/api/api_v1/endpoints/download.py | 18 +++--- .../external/api/api_v1/endpoints/sample.py | 9 ++- .../api/api_v1/endpoints/statistics.py | 4 +- .../external/api/api_v1/endpoints/update.py | 10 ++-- NIPTool/API/external/api/deps.py | 57 +++++++------------ .../internal/api/api_v1/endpoints/insert.py | 3 +- NIPTool/API/internal/api/deps.py | 12 +++- NIPTool/config.py | 20 +++++++ NIPTool/crud/find.py | 17 ++++-- requirements.txt | 1 + 12 files changed, 89 insertions(+), 76 deletions(-) create mode 100644 NIPTool/config.py diff --git a/.gitignore b/.gitignore index 4817b43b..402f640b 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ celerybeat-schedule # dotenv .env +NIPTool/.env # virtualenv .venv diff --git a/NIPTool/API/external/api/api_v1/endpoints/batches.py b/NIPTool/API/external/api/api_v1/endpoints/batches.py index 4d1716f5..24006d86 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/batches.py +++ b/NIPTool/API/external/api/api_v1/endpoints/batches.py @@ -1,12 +1,11 @@ from fastapi import APIRouter, Depends, Request +from fastapi.templating import Jinja2Templates from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.API.external.constants import TRISOMI_TRESHOLDS +from NIPTool.API.external.utils import * +from NIPTool.config import get_nipt_adapter from NIPTool.crud import find from NIPTool.models.database import Batch, User -from NIPTool.API.external.utils import * -from NIPTool.API.external.constants import TRISOMI_TRESHOLDS -from NIPTool.API.external.api.deps import get_nipt_adapter - -from fastapi.templating import Jinja2Templates router = APIRouter() templates = Jinja2Templates(directory="templates") @@ -18,7 +17,6 @@ def batches( request: Request, adapter: NiptAdapter = Depends(get_nipt_adapter) ): # , user: User = Depends(get_current_active_user)): """List of all batches""" - all_batches: List[Batch] = find.batches(adapter=adapter) return templates.TemplateResponse( "batches.html", @@ -36,7 +34,8 @@ def batches( request: Request, adapter: NiptAdapter = Depends(get_nipt_adapter) ): # , user: User = Depends(get_current_active_user)): """List of all batches""" - + print("hej") + print(adapter.client) all_batches: List[Batch] = find.batches(adapter=adapter) return templates.TemplateResponse( "batches.html", diff --git a/NIPTool/API/external/api/api_v1/endpoints/download.py b/NIPTool/API/external/api/api_v1/endpoints/download.py index b333d867..a1c43643 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/download.py +++ b/NIPTool/API/external/api/api_v1/endpoints/download.py @@ -1,10 +1,12 @@ +from pathlib import Path + from fastapi import APIRouter, Depends, Request -from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.API.external.api.deps import get_nipt_adapter, get_current_active_user +from fastapi.responses import FileResponse, RedirectResponse from fastapi.templating import Jinja2Templates +from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.config import get_nipt_adapter +from NIPTool.crud import find from NIPTool.parse.batch import validate_file_path -from pathlib import Path -from fastapi.responses import RedirectResponse, FileResponse router = APIRouter() templates = Jinja2Templates(directory="templates") @@ -16,11 +18,11 @@ def batch_download( ): """View for batch downloads""" - batch = adapter.batch(batch_id) + batch: dict = find.batch(adapter=adapter, batch_id=batch_id).dict() file_path = batch.get(file_id) if not validate_file_path(file_path): - # handle the redirect responce! + # handle the redirect response! return RedirectResponse(request.url) path = Path(file_path) @@ -36,10 +38,10 @@ def sample_download( ): """View for sample downloads""" - sample = adapter.sample(sample_id) + sample: dict = find.sample(adapter=adapter, sample_id=sample_id).dict() file_path = sample.get(file_id) if not validate_file_path(file_path): - # handle the redirect responce! + # handle the redirect response! return RedirectResponse(request.url) file = Path(file_path) diff --git a/NIPTool/API/external/api/api_v1/endpoints/sample.py b/NIPTool/API/external/api/api_v1/endpoints/sample.py index bfddfd60..c4a3f459 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/sample.py +++ b/NIPTool/API/external/api/api_v1/endpoints/sample.py @@ -1,11 +1,10 @@ from fastapi import APIRouter, Depends, Request +from fastapi.templating import Jinja2Templates from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.crud import find -from NIPTool.models.database import User from NIPTool.API.external.utils import * -from NIPTool.API.external.api.deps import get_nipt_adapter - -from fastapi.templating import Jinja2Templates +from NIPTool.config import get_nipt_adapter +from NIPTool.crud import find +from NIPTool.models.database import Batch, User router = APIRouter() templates = Jinja2Templates(directory="templates") diff --git a/NIPTool/API/external/api/api_v1/endpoints/statistics.py b/NIPTool/API/external/api/api_v1/endpoints/statistics.py index 8fece86c..7ccf407a 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/statistics.py +++ b/NIPTool/API/external/api/api_v1/endpoints/statistics.py @@ -1,5 +1,3 @@ -from typing import List - from fastapi import APIRouter, Depends, Request from NIPTool.adapter.plugin import NiptAdapter from NIPTool.models.database import User @@ -8,7 +6,7 @@ get_statistics_for_box_plot, get_statistics_for_scatter_plot, ) -from NIPTool.API.external.api.deps import get_nipt_adapter +from NIPTool.config import get_nipt_adapter from fastapi.templating import Jinja2Templates diff --git a/NIPTool/API/external/api/api_v1/endpoints/update.py b/NIPTool/API/external/api/api_v1/endpoints/update.py index 220b84e6..7cd51882 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/update.py +++ b/NIPTool/API/external/api/api_v1/endpoints/update.py @@ -1,17 +1,15 @@ import logging +from datetime import datetime from typing import Iterable from fastapi import APIRouter, Depends, Request from fastapi.responses import RedirectResponse -from starlette.datastructures import FormData - from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.API.external.utils import * +from NIPTool.config import get_nipt_adapter from NIPTool.crud import update - from NIPTool.models.database import User -from NIPTool.API.external.utils import * -from NIPTool.API.external.api.deps import get_nipt_adapter, get_current_active_user -from datetime import datetime +from starlette.datastructures import FormData router = APIRouter() diff --git a/NIPTool/API/external/api/deps.py b/NIPTool/API/external/api/deps.py index d6c4b33b..381392ab 100644 --- a/NIPTool/API/external/api/deps.py +++ b/NIPTool/API/external/api/deps.py @@ -1,23 +1,14 @@ -from pydantic import BaseSettings - -from NIPTool.adapter.plugin import NiptAdapter -from pymongo import MongoClient -from fastapi.security import OAuth2PasswordBearer -from fastapi import HTTPException, status -from NIPTool.models.server.login import User, UserInDB, TokenData -from fastapi import Depends -from passlib.context import CryptContext from datetime import datetime, timedelta from typing import Optional -from jose import JWTError, jwt - - -class Settings(BaseSettings): - db_uri: str = "test_uri" - db_name: str = "test_db" - -settings = Settings() +from fastapi import Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer +from jose import JWTError, jwt +from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.config import get_nipt_adapter +from NIPTool.crud import find +from NIPTool.models.server.login import TokenData, User, UserInDB +from passlib.context import CryptContext def temp_get_config(): @@ -30,11 +21,6 @@ def temp_get_config(): } -def get_nipt_adapter(): - client = MongoClient(settings.db_uri) - return NiptAdapter(client, db_name=settings.db_name) - - oauth2_scheme = OAuth2PasswordBearer( tokenUrl="/api/v1/login/token", scopes={"me": "Read information about the current user.", "items": "Read items."}, @@ -61,10 +47,10 @@ def get_current_user( token_data = TokenData(username=username) except JWTError: raise credentials_exception - user = adapter.user(token_data.username) + user: User = find.user(adapter=adapter, user_name=token_data.username) if not user: raise credentials_exception - return User(**user) + return user def get_current_active_user(current_user: User = Depends(get_current_user)) -> User: @@ -81,26 +67,25 @@ def get_password_hash(password): return pwd_context.hash(password) -def authenticate_user(username: str, password: str) -> UserInDB: +def authenticate_user(username: str, password: str) -> Optional[UserInDB]: adapter = get_nipt_adapter() - user_dict = adapter.user(username) - if not user_dict: - return False - user = UserInDB(**user_dict) + user: User = find.user(adapter=adapter, user_name=username) if not user: - return False - if not verify_password(password, user.hashed_password): - return False - return user + return None + db_user = UserInDB(**user.dict()) + if not db_user: + return None + if not verify_password(password, db_user.hashed_password): + return None + return db_user def create_access_token(data: dict, expires_delta: Optional[timedelta] = None): - config: dict = temp_get_config() + configs: dict = temp_get_config() to_encode = data.copy() if expires_delta: expire = datetime.utcnow() + expires_delta else: expire = datetime.utcnow() + timedelta(minutes=15) to_encode.update({"exp": expire}) - encoded_jwt = jwt.encode(to_encode, config.get("SECRET_KEY"), algorithm=config.get("ALGORITHM")) - return encoded_jwt + return jwt.encode(to_encode, configs.get("SECRET_KEY"), algorithm=configs.get("ALGORITHM")) diff --git a/NIPTool/API/internal/api/api_v1/endpoints/insert.py b/NIPTool/API/internal/api/api_v1/endpoints/insert.py index b236489c..c03800bf 100644 --- a/NIPTool/API/internal/api/api_v1/endpoints/insert.py +++ b/NIPTool/API/internal/api/api_v1/endpoints/insert.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Response, status from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.API.internal.api.deps import get_nipt_adapter +from NIPTool.config import get_nipt_adapter from NIPTool.crud import find from NIPTool.crud.insert import insert_batch, insert_samples, insert_user from NIPTool.models.database import Batch, Sample @@ -20,7 +20,6 @@ def batch( adapter: NiptAdapter = Depends(get_nipt_adapter), ): """Function to load batch data into the database with rest""" - nipt_results = Path(batch_files.result_file) if not nipt_results.exists(): response.status_code = status.HTTP_422_UNPROCESSABLE_ENTITY diff --git a/NIPTool/API/internal/api/deps.py b/NIPTool/API/internal/api/deps.py index 25570662..1145a41f 100644 --- a/NIPTool/API/internal/api/deps.py +++ b/NIPTool/API/internal/api/deps.py @@ -1,11 +1,14 @@ from NIPTool.adapter.plugin import NiptAdapter -from pymongo import MongoClient from pydantic import BaseSettings +from pymongo import MongoClient class Settings(BaseSettings): - db_uri: str = "test_uri" - db_name: str = "test_db" + db_uri: str = "test-uri" + db_name: str = "test-db" + + class Config: + env_file = ".env" settings = Settings() @@ -13,4 +16,7 @@ class Settings(BaseSettings): def get_nipt_adapter(): client = MongoClient(settings.db_uri) + print(settings) + print(settings.db_uri) + print(client) return NiptAdapter(client, db_name=settings.db_name) diff --git a/NIPTool/config.py b/NIPTool/config.py new file mode 100644 index 00000000..de3b00db --- /dev/null +++ b/NIPTool/config.py @@ -0,0 +1,20 @@ +from pydantic import BaseSettings +from pymongo import MongoClient + +from NIPTool.adapter.plugin import NiptAdapter + + +class Settings(BaseSettings): + db_uri: str = "test_uri" + db_name: str = "test_db" + + class Config: + from_file = ".env" + + +settings = Settings() + + +def get_nipt_adapter(): + client = MongoClient(settings.db_uri) + return NiptAdapter(client, db_name=settings.db_name) diff --git a/NIPTool/crud/find.py b/NIPTool/crud/find.py index c0cbac41..aaf259e5 100644 --- a/NIPTool/crud/find.py +++ b/NIPTool/crud/find.py @@ -1,15 +1,20 @@ from typing import Iterable, List, Optional -from pydantic import parse_obj_as - from NIPTool.adapter import NiptAdapter -from NIPTool.models.database import Batch, User, Sample +from NIPTool.models.database import Batch, Sample, User +from pydantic import parse_obj_as -def user(adapter: NiptAdapter, email: str) -> Optional[User]: +def user( + adapter: NiptAdapter, email: Optional[str] = None, user_name: Optional[str] = None +) -> Optional[User]: """Find user from user collection""" - - raw_user: dict = adapter.user_collection.find_one({"email": email}) # ????? wierd + if email: + raw_user: dict = adapter.user_collection.find_one({"email": email}) # ????? wierd + elif user_name: + raw_user: dict = adapter.user_collection.find_one({"username": user_name}) + else: + raise SyntaxError("Have to use email or user_name") if not raw_user: return None diff --git a/requirements.txt b/requirements.txt index 4610c1f0..16316d6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ pyyaml cerberus python-multipart +python-dotenv # server stuff passlib From aca690503fcefbbad52de7d42ab9f5f9359381fe Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 15:07:46 +0200 Subject: [PATCH 3/9] Remove unused file --- NIPTool/API/internal/api/deps.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 NIPTool/API/internal/api/deps.py diff --git a/NIPTool/API/internal/api/deps.py b/NIPTool/API/internal/api/deps.py deleted file mode 100644 index 1145a41f..00000000 --- a/NIPTool/API/internal/api/deps.py +++ /dev/null @@ -1,22 +0,0 @@ -from NIPTool.adapter.plugin import NiptAdapter -from pydantic import BaseSettings -from pymongo import MongoClient - - -class Settings(BaseSettings): - db_uri: str = "test-uri" - db_name: str = "test-db" - - class Config: - env_file = ".env" - - -settings = Settings() - - -def get_nipt_adapter(): - client = MongoClient(settings.db_uri) - print(settings) - print(settings.db_uri) - print(client) - return NiptAdapter(client, db_name=settings.db_name) From 2b08b42603b9f37743748207045eddbaebdfec16 Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 20:25:22 +0200 Subject: [PATCH 4/9] Move templates to config.py --- .../external/api/api_v1/endpoints/batches.py | 7 ++---- NIPTool/commands/base.py | 8 +++--- NIPTool/config.py | 7 +++++- NIPTool/main.py | 25 ++++++++++++++++++- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/NIPTool/API/external/api/api_v1/endpoints/batches.py b/NIPTool/API/external/api/api_v1/endpoints/batches.py index 24006d86..91e30974 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/batches.py +++ b/NIPTool/API/external/api/api_v1/endpoints/batches.py @@ -1,14 +1,12 @@ from fastapi import APIRouter, Depends, Request -from fastapi.templating import Jinja2Templates from NIPTool.adapter.plugin import NiptAdapter from NIPTool.API.external.constants import TRISOMI_TRESHOLDS from NIPTool.API.external.utils import * -from NIPTool.config import get_nipt_adapter +from NIPTool.config import get_nipt_adapter, templates from NIPTool.crud import find from NIPTool.models.database import Batch, User router = APIRouter() -templates = Jinja2Templates(directory="templates") CURRENT_USER = User(username="mayapapaya", email="mayabrandi@123.com", role="RW").dict() @@ -34,9 +32,8 @@ def batches( request: Request, adapter: NiptAdapter = Depends(get_nipt_adapter) ): # , user: User = Depends(get_current_active_user)): """List of all batches""" - print("hej") - print(adapter.client) all_batches: List[Batch] = find.batches(adapter=adapter) + return templates.TemplateResponse( "batches.html", context={ diff --git a/NIPTool/commands/base.py b/NIPTool/commands/base.py index 6ccfe330..6531233e 100644 --- a/NIPTool/commands/base.py +++ b/NIPTool/commands/base.py @@ -4,7 +4,8 @@ from typing import List import click -from dotenv import dotenv_values, load_dotenv +import pkg_resources +from dotenv import dotenv_values # Get version and doc decorator from NIPTool import __version__ @@ -19,6 +20,8 @@ LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] LOG = logging.getLogger(__name__) +ENV_FILE = pkg_resources.resource_filename("NIPTool", ".env") + @click.version_option(__version__) @click.group() @@ -26,8 +29,7 @@ def cli(context: click.Context): """ Main entry point """ logging.basicConfig(level=logging.INFO) - load_dotenv() - settings: dict = dotenv_values(".env") + settings: dict = dotenv_values(ENV_FILE) client = MongoClient(settings["DB_URI"]) context.obj = {"adapter": NiptAdapter(client, db_name=settings["DB_NAME"])} LOG.info("Connected to %s", settings["DB_NAME"]) diff --git a/NIPTool/config.py b/NIPTool/config.py index de3b00db..4f38e5e7 100644 --- a/NIPTool/config.py +++ b/NIPTool/config.py @@ -1,3 +1,5 @@ +import pkg_resources +from fastapi.templating import Jinja2Templates from pydantic import BaseSettings from pymongo import MongoClient @@ -9,12 +11,15 @@ class Settings(BaseSettings): db_name: str = "test_db" class Config: - from_file = ".env" + from_file = pkg_resources.resource_filename("NIPTool", ".env") settings = Settings() +templates = Jinja2Templates(directory="./NIPTool/API/external/api/api_v1/templates") def get_nipt_adapter(): + print("hej") + print(settings) client = MongoClient(settings.db_uri) return NiptAdapter(client, db_name=settings.db_name) diff --git a/NIPTool/main.py b/NIPTool/main.py index e331b8ef..5cdd10fb 100644 --- a/NIPTool/main.py +++ b/NIPTool/main.py @@ -1 +1,24 @@ -from NIPTool.API.external.api.api_v1.api import app +from fastapi import FastAPI + +from NIPTool.API.external.api.api_v1.endpoints import ( + batches, + download, + index, + login, + sample, + statistics, + update, +) +from NIPTool.API.internal.api.api_v1.endpoints import insert + +external_app = FastAPI() +external_app.include_router(login.router, prefix="/api/v1/login", tags=["login"]) +external_app.include_router(batches.router, prefix="/api/v1/batches", tags=["batches"]) +external_app.include_router(index.router, prefix="/api/v1", tags=["index"]) +external_app.include_router(sample.router, prefix="/api/v1", tags=["sample"]) +external_app.include_router(update.router, prefix="/api/v1", tags=["update"]) +external_app.include_router(download.router, prefix="/api/v1", tags=["download"]) +external_app.include_router(statistics.router, prefix="/api/v1", tags=["statistics"]) + +internal_app = FastAPI() +internal_app.include_router(insert.router, prefix="/api/v1/insert", tags=["insert"]) From 048aa48e1401170dbe67ccac25aa740c715cdc2b Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 20:26:30 +0200 Subject: [PATCH 5/9] Remove api and move apps to main --- NIPTool/API/external/api/api_v1/api.py | 21 --------------------- NIPTool/API/internal/api/api_v1/api.py | 8 -------- 2 files changed, 29 deletions(-) delete mode 100644 NIPTool/API/external/api/api_v1/api.py delete mode 100644 NIPTool/API/internal/api/api_v1/api.py diff --git a/NIPTool/API/external/api/api_v1/api.py b/NIPTool/API/external/api/api_v1/api.py deleted file mode 100644 index a774ba8c..00000000 --- a/NIPTool/API/external/api/api_v1/api.py +++ /dev/null @@ -1,21 +0,0 @@ -from fastapi import FastAPI - -from NIPTool.API.external.api.api_v1.endpoints import ( - batches, - index, - sample, - update, - download, - statistics, - login, -) - -app = FastAPI() - -app.include_router(login.router, prefix="/api/v1/login", tags=["login"]) -app.include_router(batches.router, prefix="/api/v1/batches", tags=["batches"]) -app.include_router(index.router, prefix="/api/v1", tags=["index"]) -app.include_router(sample.router, prefix="/api/v1", tags=["sample"]) -app.include_router(update.router, prefix="/api/v1", tags=["update"]) -app.include_router(download.router, prefix="/api/v1", tags=["download"]) -app.include_router(statistics.router, prefix="/api/v1", tags=["statistics"]) diff --git a/NIPTool/API/internal/api/api_v1/api.py b/NIPTool/API/internal/api/api_v1/api.py deleted file mode 100644 index b60ffe66..00000000 --- a/NIPTool/API/internal/api/api_v1/api.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import FastAPI - -from NIPTool.API.internal.api.api_v1.endpoints import insert - - -app = FastAPI() - -app.include_router(insert.router, prefix="/api/v1/insert", tags=["insert"]) From 697f8599d512ccaa290d7c3dad912ca53a46c44f Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 21:04:47 +0200 Subject: [PATCH 6/9] Adds serve command to nipt cli --- NIPTool/commands/base.py | 28 ++++++++++++++++++++-------- NIPTool/config.py | 2 ++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/NIPTool/commands/base.py b/NIPTool/commands/base.py index 6531233e..d4fbde6d 100644 --- a/NIPTool/commands/base.py +++ b/NIPTool/commands/base.py @@ -5,13 +5,16 @@ import click import pkg_resources +import uvicorn from dotenv import dotenv_values # Get version and doc decorator from NIPTool import __version__ from NIPTool.adapter import NiptAdapter +from NIPTool.config import settings from NIPTool.crud import find from NIPTool.crud.insert import insert_batch, insert_samples +from NIPTool.main import external_app, internal_app from NIPTool.models.database import Batch, Sample from NIPTool.models.server.load import BatchRequestBody from NIPTool.parse.batch import get_batch, get_samples @@ -25,14 +28,9 @@ @click.version_option(__version__) @click.group() -@click.pass_context -def cli(context: click.Context): +def cli(): """ Main entry point """ logging.basicConfig(level=logging.INFO) - settings: dict = dotenv_values(ENV_FILE) - client = MongoClient(settings["DB_URI"]) - context.obj = {"adapter": NiptAdapter(client, db_name=settings["DB_NAME"])} - LOG.info("Connected to %s", settings["DB_NAME"]) @cli.command(name="load") @@ -49,12 +47,26 @@ def load_command( multiqc_report=str(multiqc_report), segmental_calls=str(segmental_calls), ) - + client = MongoClient(settings.db_uri) + adapter = NiptAdapter(client, db_name=settings.db_name) + LOG.info("Connected to %s", settings.db_name) nipt_results = Path(str(result_file)) - adapter: NiptAdapter = context["adapter"] samples: List[Sample] = get_samples(nipt_results) batch: Batch = get_batch(nipt_results) if find.batch(adapter=adapter, batch_id=batch.batch_id): return "batch already in database" insert_batch(adapter=adapter, batch=batch, batch_files=batch_files) insert_samples(adapter=adapter, samples=samples, segmental_calls=batch_files.segmental_calls) + + +@cli.command(name="serve") +@click.option( + "--api", default="external", type=click.Choice(["external", "internal"]), show_default=True +) +@click.option("--reload", is_flag=True) +def serve_command(reload: bool, api: str): + app = "NIPTool.main:external_app" + if api == "internal": + app = "NIPTool.main:internal_app" + LOG.info("Running %s api on host:%s and port:%s", api, settings.host, settings.port) + uvicorn.run(app=app, host=settings.host, port=settings.port, reload=reload) diff --git a/NIPTool/config.py b/NIPTool/config.py index 4f38e5e7..b698c48b 100644 --- a/NIPTool/config.py +++ b/NIPTool/config.py @@ -9,6 +9,8 @@ class Settings(BaseSettings): db_uri: str = "test_uri" db_name: str = "test_db" + host: str = "localhost" + port: int = 8000 class Config: from_file = pkg_resources.resource_filename("NIPTool", ".env") From 8034750248428bada4a206be7410d38594bc42c7 Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 21:08:24 +0200 Subject: [PATCH 7/9] Updates tests --- tests/conftest.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d5a86ed0..fcd4552b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,15 @@ from pathlib import Path + import pytest +from fastapi.testclient import TestClient from mongomock import MongoClient -from .small_helpers import SmallHelpers from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.API.internal.api.api_v1.api import app -from NIPTool.API.internal.api.deps import get_nipt_adapter -from fastapi.testclient import TestClient +from NIPTool.config import get_nipt_adapter +from NIPTool.main import internal_app as app from NIPTool.models.server.load import BatchRequestBody, UserRequestBody +from .small_helpers import SmallHelpers + DATABASE = "testdb" @@ -16,8 +18,7 @@ def override_nipt_adapter(): mongo_client = MongoClient() database = mongo_client[DATABASE] - adapter = NiptAdapter(database.client, db_name=DATABASE) - return adapter + return NiptAdapter(database.client, db_name=DATABASE) @pytest.fixture() From a59936b0caaa674c12c556402206df4e8f060a87 Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 21:24:25 +0200 Subject: [PATCH 8/9] Update README --- NIPTool/config.py | 4 ++-- README.md | 50 ++++++++++++++++++-------------------------- requirements-dev.txt | 4 +--- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/NIPTool/config.py b/NIPTool/config.py index b698c48b..e9ca04c7 100644 --- a/NIPTool/config.py +++ b/NIPTool/config.py @@ -7,8 +7,8 @@ class Settings(BaseSettings): - db_uri: str = "test_uri" - db_name: str = "test_db" + db_uri: str = "mongodb://localhost:27017/nipt-demo" + db_name: str = "nipt-demo" host: str = "localhost" port: int = 8000 diff --git a/README.md b/README.md index 3a20de3c..9b6aedf6 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ # NIPTool [![Coverage Status](https://coveralls.io/repos/github/Clinical-Genomics/NIPTool/badge.svg?branch=master)](https://coveralls.io/github/Clinical-Genomics/NIPTool?branch=master) [![Build Status](https://travis-ci.org/Clinical-Genomics/NIPTool.svg?branch=master)](https://travis-ci.org/Clinical-Genomics/NIPTool) ![Latest Release](https://img.shields.io/github/v/release/clinical-genomics/NIPTool) -NIPTool is a visualisation tool for NIPT data. +NIPTool is a visualisation tool for the data produced by the [Fluffy] pipeline running [WisecondorX] to analyze NIPT. ## Installation - ```bash -git clone https://github.com/Clinical-Genomics/NIPTool.git +git clone https://github.com/Clinical-Genomics/NIPTool cd NIPTool pip install -r requirements.txt -e . ``` @@ -16,23 +15,26 @@ pip install -r requirements.txt -e . ### Demo -Once installed, you can setup NIPTool by running a few commands using the included command line interface. Given you have a MongoDB server listening on the default port (27017), this is how you would setup a fully working NIPTool demo: +Once installed, you can set up NIPTool by running a few commands using the included command line interface. +Given you have a MongoDB server listening on the default port (27017), this is how you would set up a fully working +NIPTool demo: ```bash -nipt -c NIPTool/tests/fixtures/nipt_config.yaml load batch -b NIPTool/tests/fixtures/valid_fluffy.csv -nipt -c NIPTool/tests/fixtures/nipt_config.yaml load user -n -r RW -e +nipt load batch --result-file NIPTool/tests/fixtures/valid_fluffy.csv ``` -This will setup an instance of NIPTool with a database called `nipt-demo`. Now run +Settings can be used by exporting the environment variables: `DB_NAME`, `DB_URI`, `HOST`, `PORT` +This will set up an instance of NIPTool with a database called `nipt-demo`. Now run ```bash -nipt run +nipt serve --reload ``` -And play around with the interface. + and play around with the interface. ### Docker image -NIPTool can be runned also as a container. The image is available [on Docker Hub](https://hub.docker.com/repository/docker/clinicalgenomics/niptool) or can be build using the Dockerfile provided in this repository. +NIPTool can also run as a container. The image is available [on Docker Hub][docker-hub] or can be build using the +Dockerfile provided in this repository. To build a new image from the Dockerfile use the commands: `docker build -t niptool .` @@ -40,9 +42,9 @@ To run the image use the following command: `docker run --name niptool niptool n To remove the container, type: `docker rm niptool` - ## Release model -NIPTool development is organised on a flexible Git "Release Flow" branching system. This more or less means that we make releases in release branches which corresponds to stable versions of NIPTool. +NIPTool development is organised on a flexible Git "Release Flow" branching system. This more or less means that we +make releases in release branches which corresponds to stable versions of NIPTool. ### Steps to make a new release: @@ -63,7 +65,8 @@ NIPTool development is organised on a flexible Git "Release Flow" branching syst ### Deploying to production -Use `update-nipt-prod.sh` script to update production both on Hasta and clinical-db. **Please follow the development guide and `servers` repo when doing so. It is also important to keep those involved informed.** +Use `update-nipt-prod.sh` script to update production both on Hasta and clinical-db. +**Please follow the development guide and `servers` repo when doing so. It is also important to keep those involved informed.** ## Back End The NIPT database is a Mongo database consisting of following collections: @@ -72,22 +75,9 @@ The NIPT database is a Mongo database consisting of following collections: - **sample** - holds sample level information. - **user** - holds user names, emails and roles. -The database is loaded through the CLI with data generated by the [FluFFyPipe](https://github.com/Clinical-Genomics/fluffy). - -## CLI -The CLI has two base commands - load and run. The load command is for loading batch and sample data into the nipt database, and the run command is for running the web application. - -### Load - - -``` -Usage: nipt -c load batch [OPTIONS] ... -``` - +The database is loaded through the CLI with data generated by the [FluFFyPipe][Fluffy] -### Run -``` -Usage: nipt run [OPTIONS] ... - -``` +[Fluffy]: https://github.com/Clinical-Genomics/fluffy +[WisecondorX]: https://github.com/CenterForMedicalGeneticsGhent/WisecondorX +[docker-hub]: https://hub.docker.com/repository/docker/clinicalgenomics/niptool \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 1e1c3006..c32e68e0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,5 @@ -flask_debugtoolbar -pytest-flask pytest>=5.2 pytest-cov coveralls pylint -werkzeug<1.0.0 # due to breaking changes in 1.0.0 +mongomock \ No newline at end of file From fc4471e3a2e0d4589e1c34193246ba60650e7f34 Mon Sep 17 00:00:00 2001 From: moonso Date: Thu, 1 Apr 2021 21:35:04 +0200 Subject: [PATCH 9/9] Use base templates everywhere --- .../external/api/api_v1/endpoints/download.py | 4 +--- .../API/external/api/api_v1/endpoints/index.py | 5 +---- .../API/external/api/api_v1/endpoints/login.py | 17 +++++------------ .../API/external/api/api_v1/endpoints/sample.py | 8 +++----- .../external/api/api_v1/endpoints/statistics.py | 7 ++----- .../API/internal/api/api_v1/endpoints/insert.py | 2 +- NIPTool/commands/base.py | 2 -- 7 files changed, 13 insertions(+), 32 deletions(-) diff --git a/NIPTool/API/external/api/api_v1/endpoints/download.py b/NIPTool/API/external/api/api_v1/endpoints/download.py index a1c43643..a6cb0b03 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/download.py +++ b/NIPTool/API/external/api/api_v1/endpoints/download.py @@ -2,14 +2,12 @@ from fastapi import APIRouter, Depends, Request from fastapi.responses import FileResponse, RedirectResponse -from fastapi.templating import Jinja2Templates from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.config import get_nipt_adapter +from NIPTool.config import get_nipt_adapter, templates from NIPTool.crud import find from NIPTool.parse.batch import validate_file_path router = APIRouter() -templates = Jinja2Templates(directory="templates") @router.get("/batch_download/{batch_id}/{file_id}") diff --git a/NIPTool/API/external/api/api_v1/endpoints/index.py b/NIPTool/API/external/api/api_v1/endpoints/index.py index def32205..3061fa85 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/index.py +++ b/NIPTool/API/external/api/api_v1/endpoints/index.py @@ -1,8 +1,5 @@ from fastapi import APIRouter, Request -from fastapi.templating import Jinja2Templates - - -templates = Jinja2Templates(directory="templates") +from NIPTool.config import templates router = APIRouter() diff --git a/NIPTool/API/external/api/api_v1/endpoints/login.py b/NIPTool/API/external/api/api_v1/endpoints/login.py index 83c538ed..b7ac759f 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/login.py +++ b/NIPTool/API/external/api/api_v1/endpoints/login.py @@ -1,17 +1,10 @@ -from fastapi import APIRouter, Response -from fastapi.responses import RedirectResponse +from datetime import timedelta -from fastapi import Depends, HTTPException, status, Security +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.responses import RedirectResponse from fastapi.security import OAuth2PasswordRequestForm -from NIPTool.models.server.login import Token, UserInDB, User - -from NIPTool.API.external.api.deps import ( - get_current_active_user, - authenticate_user, - create_access_token, - temp_get_config, -) -from datetime import timedelta +from NIPTool.API.external.api.deps import authenticate_user, create_access_token, temp_get_config +from NIPTool.models.server.login import Token, UserInDB router = APIRouter() diff --git a/NIPTool/API/external/api/api_v1/endpoints/sample.py b/NIPTool/API/external/api/api_v1/endpoints/sample.py index c4a3f459..8000c4be 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/sample.py +++ b/NIPTool/API/external/api/api_v1/endpoints/sample.py @@ -1,18 +1,16 @@ from fastapi import APIRouter, Depends, Request -from fastapi.templating import Jinja2Templates from NIPTool.adapter.plugin import NiptAdapter from NIPTool.API.external.utils import * -from NIPTool.config import get_nipt_adapter +from NIPTool.config import get_nipt_adapter, templates from NIPTool.crud import find from NIPTool.models.database import Batch, User router = APIRouter() -templates = Jinja2Templates(directory="templates") @router.get("/samples/{sample_id}/") def sample(request: Request, sample_id: str, adapter: NiptAdapter = Depends(get_nipt_adapter)): - """Sample view with sample information.""" + """Get sample with id""" sample: dict = find.sample(sample_id=sample_id, adapter=adapter).dict() batch: Batch = find.batch(batch_id=sample.get("batch_id"), adapter=adapter) @@ -33,7 +31,7 @@ def sample(request: Request, sample_id: str, adapter: NiptAdapter = Depends(get_ @router.post("/samples/{sample_id}/") def sample(request: Request, sample_id: str, adapter: NiptAdapter = Depends(get_nipt_adapter)): - """Sample view with sample information.""" + """Post sample with id""" sample: dict = find.sample(sample_id=sample_id, adapter=adapter).dict() batch: Batch = find.batch(batch_id=sample.get("batch_id"), adapter=adapter) diff --git a/NIPTool/API/external/api/api_v1/endpoints/statistics.py b/NIPTool/API/external/api/api_v1/endpoints/statistics.py index 7ccf407a..09d5dbc8 100644 --- a/NIPTool/API/external/api/api_v1/endpoints/statistics.py +++ b/NIPTool/API/external/api/api_v1/endpoints/statistics.py @@ -1,17 +1,14 @@ from fastapi import APIRouter, Depends, Request from NIPTool.adapter.plugin import NiptAdapter -from NIPTool.models.database import User from NIPTool.API.external.utils import ( get_last_batches, get_statistics_for_box_plot, get_statistics_for_scatter_plot, ) -from NIPTool.config import get_nipt_adapter - -from fastapi.templating import Jinja2Templates +from NIPTool.config import get_nipt_adapter, templates +from NIPTool.models.database import User router = APIRouter() -templates = Jinja2Templates(directory="templates") @router.get("/statistics") diff --git a/NIPTool/API/internal/api/api_v1/endpoints/insert.py b/NIPTool/API/internal/api/api_v1/endpoints/insert.py index c03800bf..d7634efb 100644 --- a/NIPTool/API/internal/api/api_v1/endpoints/insert.py +++ b/NIPTool/API/internal/api/api_v1/endpoints/insert.py @@ -43,7 +43,7 @@ def user( """Function to load user into the database with rest""" if find.user(adapter=adapter, email=user.email): - return "user allready in database" + return "user already in database" insert_user(adapter=adapter, user=user) response.status_code = status.HTTP_200_OK diff --git a/NIPTool/commands/base.py b/NIPTool/commands/base.py index d4fbde6d..2901670d 100644 --- a/NIPTool/commands/base.py +++ b/NIPTool/commands/base.py @@ -6,7 +6,6 @@ import click import pkg_resources import uvicorn -from dotenv import dotenv_values # Get version and doc decorator from NIPTool import __version__ @@ -14,7 +13,6 @@ from NIPTool.config import settings from NIPTool.crud import find from NIPTool.crud.insert import insert_batch, insert_samples -from NIPTool.main import external_app, internal_app from NIPTool.models.database import Batch, Sample from NIPTool.models.server.load import BatchRequestBody from NIPTool.parse.batch import get_batch, get_samples