diff --git a/NIPTool/build/document.py b/NIPTool/build/document.py index fb54d138..11f63c61 100644 --- a/NIPTool/build/document.py +++ b/NIPTool/build/document.py @@ -23,9 +23,16 @@ def build_sample(sample_data: dict) -> dict: return sample -def build_batch(batch_data: dict) -> dict: +def build_batch(batch_data: dict, request_data: dict) -> dict: """Builds a document for the batch collection""" batch = build_document(batch_data, BATCH_KEYS) + batch["_id"] = request_data['project_name'] + batch["fluffy_result_file"] = request_data['result_file'] + + if request_data.get('multiqc_report'): + batch["multiqc_report"] = request_data['multiqc_report'] + if request_data.get('segmental_calls'): + batch["segmental_calls"] = request_data['segmental_calls'] return batch diff --git a/NIPTool/commands/base.py b/NIPTool/commands/base.py index 94d5db8f..a271a6ce 100644 --- a/NIPTool/commands/base.py +++ b/NIPTool/commands/base.py @@ -2,7 +2,6 @@ import logging import click -import coloredlogs from flask.cli import FlaskGroup, with_appcontext from flask import current_app @@ -12,8 +11,6 @@ # Get version and doc decorator from NIPTool import __version__ -from NIPTool.tools.cli_utils import add_doc as doc -from .load import load LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] LOG = logging.getLogger(__name__) @@ -53,4 +50,3 @@ def name(): cli.add_command(test) cli.add_command(name) -cli.add_command(load) diff --git a/NIPTool/commands/load/__init__.py b/NIPTool/commands/load/__init__.py deleted file mode 100644 index 8ef2f5a9..00000000 --- a/NIPTool/commands/load/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base import load diff --git a/NIPTool/commands/load/base.py b/NIPTool/commands/load/base.py deleted file mode 100755 index 89f24d41..00000000 --- a/NIPTool/commands/load/base.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -import click - -# commands -from NIPTool.commands.load.batch import batch as load_batch_cmd -from NIPTool.commands.load.user import user as load_user_cmd - -# Get version -from NIPTool import __version__ - - -@click.group() -@click.version_option(version=__version__) -def load(): - """Main entry point of load commands""" - pass - - -load.add_command(load_batch_cmd) -load.add_command(load_user_cmd) diff --git a/NIPTool/commands/load/batch.py b/NIPTool/commands/load/batch.py deleted file mode 100644 index 00e0ed69..00000000 --- a/NIPTool/commands/load/batch.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging -import click -from NIPTool.load.batch import load_result_file, load_concentrastions -from flask.cli import with_appcontext, current_app -from datetime import date, timedelta -from NIPTool.exeptions import NIPToolError -import json -from pathlib import Path -from NIPTool.exeptions import MissingResultsError - - -LOG = logging.getLogger(__name__) - - -@click.command("batch", short_help="load batch into db.") -@click.option("-b", "--load-config", help="path to batch load config") -@with_appcontext -def batch(load_config: dict) -> None: - """Read and load data for one batch. - - Args: load_config - dict with keys: - "concentrations" - "result_file" - "project_name" - """ - - file = Path(load_config) - - if not file.exists(): - raise MissingResultsError("Results file missing.") - - with open(file) as data_file: - config_data = json.load(data_file) - - try: - load_result_file( - current_app.adapter, config_data["result_file"], config_data["project_name"] - ) - load_concentrastions(current_app.adapter, config_data["concentrations"]) - except NIPToolError as e: - LOG.error(e.message) - raise click.Abort() diff --git a/NIPTool/commands/load/user.py b/NIPTool/commands/load/user.py deleted file mode 100644 index e5443404..00000000 --- a/NIPTool/commands/load/user.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging -import click -from NIPTool.load.user import load_user -from flask.cli import with_appcontext, current_app -from NIPTool.exeptions import NIPToolError - - -LOG = logging.getLogger(__name__) - - -@click.command("user", short_help="load a user into db.") -@click.option("-n", "--name", help="User name") -@click.option("-r", "--role", help="User role") -@click.option("-e", "--email", help="User email") -@with_appcontext -def user(name, role, email): - """Loading new user to db.""" - - try: - load_user(current_app.adapter, email, name, role) - except NIPToolError as e: - LOG.error(e.message) - raise click.Abort() diff --git a/NIPTool/load/batch.py b/NIPTool/load/batch.py index b0363e98..ced00930 100644 --- a/NIPTool/load/batch.py +++ b/NIPTool/load/batch.py @@ -1,14 +1,17 @@ import logging from NIPTool.build.document import build_sample, build_batch -from NIPTool.parse.batch import parse_batch_file from NIPTool.models.validation import requiered_fields +from NIPTool.exeptions import MissingResultsError +from pathlib import Path + +import json LOG = logging.getLogger(__name__) def check_requiered_fields(document): - """Check if document keys contain requiered fields""" + """Check if document keys contain required fields""" if not set(requiered_fields).issubset(set(document.keys())): LOG.info(f"Could not add document {document}. Requiered fields missing.") @@ -16,39 +19,43 @@ def check_requiered_fields(document): return True -def load_result_file(adapter, batch_data: list, project_name: str) -> None: - """Function to load data from fluffy result file. - Raises: - MissingResultsError: when parsing file that is empty""" - - batch_data = parse_batch_file(batch_data) +def load_batch(adapter, batch_data: dict, request_data: dict) -> None: + """Function to load data from fluffy result file.""" - mongo_batch = build_batch(batch_data[0]) - mongo_batch['_id'] = project_name + mongo_batch = build_batch(batch_data, request_data) adapter.add_or_update_document(mongo_batch, adapter.batch_collection) - + + +def load_samples(adapter, batch_data: list, project_name: str) -> None: + """Function to load data from fluffy result file.""" for sample in batch_data: if not check_requiered_fields(sample): continue mongo_sample = build_sample(sample) - mongo_sample['SampleProject'] = project_name + mongo_sample["SampleProject"] = project_name adapter.add_or_update_document(mongo_sample, adapter.sample_collection) - if not check_requiered_fields(batch_data[0]): - return - -def load_concentrastions(adapter, concentrations: dict) -> None: +def load_concentrations(adapter, concentrations_file: str) -> None: """Function to load concentrations to samples in the database.""" + file = Path(concentrations_file) + + if not file.exists(): + raise MissingResultsError("Concentrations file missing.") + + with open(file) as data_file: + concentrations = json.load(data_file) + for sample, concentration in concentrations.items(): mongo_sample = adapter.sample(sample) if not mongo_sample: - LOG.warning(f"Trying to add concentration to sample {sample} but it doesnt exist in the databse.") + LOG.warning( + f"Trying to add concentration to sample {sample} but it doesnt exist in the database." + ) return mongo_sample["concentration"] = concentration adapter.add_or_update_document(mongo_sample, adapter.sample_collection) - diff --git a/NIPTool/models/validation.py b/NIPTool/models/validation.py index 8410733d..36fc54e0 100644 --- a/NIPTool/models/validation.py +++ b/NIPTool/models/validation.py @@ -89,6 +89,15 @@ "GC_Dropout", "AT_Dropout", "Bin2BinVariance", - + ] +req_str = {'type': 'string', 'required': True} +nreq_str = {'type': 'string', 'required': False} + +batch_load_schema = {'concentrations': nreq_str, + 'project_name': req_str, + 'result_file': req_str, + 'multiqc_report': nreq_str, + 'segmental_calls': nreq_str} +user_load_schema = {'email': req_str, 'name': req_str, 'role': req_str} diff --git a/NIPTool/parse/batch.py b/NIPTool/parse/batch.py index c7add708..30747e8a 100644 --- a/NIPTool/parse/batch.py +++ b/NIPTool/parse/batch.py @@ -1,10 +1,10 @@ import logging -import pandas as pd +import pandas from pathlib import Path from typing import Optional, List -from NIPTool.exeptions import MissingResultsError, FileValidationError +from NIPTool.exeptions import MissingResultsError from NIPTool.models.validation import ( ints, floats, @@ -15,8 +15,9 @@ LOG = logging.getLogger(__name__) + def form(val: Optional, function) -> Optional: - """Returning formated value or None""" + """Returning formatted value or None""" try: return function(val) @@ -25,23 +26,23 @@ def form(val: Optional, function) -> Optional: def validate(key: str, val: Optional) -> Optional: - """Formating value according to defined models.""" + """Formatting value according to defined models.""" if val in exceptions: - formated_value = None + formatted_value = None elif key in ints: - formated_value = form(val, int) + formatted_value = form(val, int) elif key in floats: - formated_value = form(val, float) + formatted_value = form(val, float) elif key in strings: - formated_value = form(val, str) + formatted_value = form(val, str) else: - formated_value = None - return formated_value + formatted_value = None + return formatted_value def parse_batch_file(nipt_results_path: str) -> List[dict]: - """Parsing file content. Formating values. Ignoring values + """Parsing file content. Formatting values. Ignoring values that could not be formatted according to defined models""" file = Path(nipt_results_path) @@ -49,19 +50,18 @@ def parse_batch_file(nipt_results_path: str) -> List[dict]: if not file.exists(): raise MissingResultsError("Results file missing.") - with open(file) as nipt_results_path: - df = pd.read_csv(file, na_filter=False) - results = df.to_dict(orient="records") + data_frame = pandas.read_csv(file, na_filter=False) + results = data_frame.to_dict(orient="records") samples = [] for sample in results: - formated_results = {} + formatted_results = {} for key, val in sample.items(): - formated_value = validate(key, val) - if formated_value is None: + formatted_value = validate(key, val) + if formatted_value is None: LOG.info(f"invalid format of {key}.") continue - formated_results[key] = formated_value - samples.append(formated_results) + formatted_results[key] = formatted_value + samples.append(formatted_results) return samples diff --git a/NIPTool/server/__init__.py b/NIPTool/server/__init__.py index 560e5611..bf263f5f 100644 --- a/NIPTool/server/__init__.py +++ b/NIPTool/server/__init__.py @@ -1,15 +1,13 @@ -import os import logging from flask import Flask from pymongo import MongoClient import yaml -from uuid import uuid4 - from NIPTool.adapter.plugin import NiptAdapter from NIPTool.server.login import login_bp, login_manager from NIPTool.server.views import server_bp +from NIPTool.server.load import load_bp logging.basicConfig(level=logging.INFO) LOG = logging.getLogger(__name__) @@ -31,7 +29,6 @@ def create_app(test=False): def configure_app(app, config=None): - if config: app.config = {**app.config, **yaml.safe_load(config)} app.config['SECRET_KEY'] = app.config['SECRET_KEY'] @@ -39,14 +36,14 @@ def configure_app(app, config=None): db_name = app.config['DB_NAME'] app.client = client app.db = client[db_name] - app.adapter = NiptAdapter(client, db_name = db_name) + app.adapter = NiptAdapter(client, db_name=db_name) app.register_blueprint(login_bp) app.register_blueprint(server_bp) + app.register_blueprint(load_bp) login_manager.init_app(app) - if app.config['DEBUG']==1: + if app.config['DEBUG'] == 1: from flask_debugtoolbar import DebugToolbarExtension toolbar = DebugToolbarExtension(app) return app - diff --git a/NIPTool/server/load.py b/NIPTool/server/load.py new file mode 100644 index 00000000..56795aff --- /dev/null +++ b/NIPTool/server/load.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +from flask import ( + jsonify, + request, + Blueprint, + current_app) + +from NIPTool.parse.batch import parse_batch_file +from NIPTool.load.batch import load_batch, load_samples, load_concentrations +from NIPTool.load.user import load_user +from NIPTool.models.validation import user_load_schema, batch_load_schema +from cerberus import Validator + +import logging + +LOG = logging.getLogger(__name__) + +app = current_app +load_bp = Blueprint("load", __name__) + + +@load_bp.route("/batch", methods=["POST"]) +def batch(): + """Function to load batch data into the database with rest""" + + request_data = request.form + v = Validator(batch_load_schema) + if not v.validate(request_data): + message = "Incomplete batch load request" + resp = jsonify({"message": message}) + resp.status_code = 400 + return resp + + batch_data = parse_batch_file(request_data['result_file']) + load_batch(current_app.adapter, batch_data[0], request_data) + load_samples(current_app.adapter, batch_data, request_data['project_name']) + load_concentrations(current_app.adapter, request_data["concentrations"]) + + message = "Data loaded into database" + resp = jsonify({"message": message}) + resp.status_code = 200 + return resp + + +@load_bp.route("/user", methods=["POST"]) +def user(): + """Function to load user into the database with rest""" + + request_data = request.form + v = Validator(user_load_schema) + if not v.validate(request_data): + message = "Incomplete user load request" + resp = jsonify({"message": message}) + resp.status_code = 400 + return resp + + load_user(current_app.adapter, request_data["email"], request_data["name"], request_data["role"]) + + message = "Data loaded into database" + resp = jsonify({"message": message}) + resp.status_code = 200 + return resp diff --git a/NIPTool/server/login.py b/NIPTool/server/login.py index c77bd7f0..b4c71367 100644 --- a/NIPTool/server/login.py +++ b/NIPTool/server/login.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from flask import url_for, redirect, request, Blueprint, session, current_app, flash +from flask import url_for, redirect, Blueprint, session, current_app, flash from flask_login import LoginManager from authlib.integrations.flask_client import OAuth diff --git a/NIPTool/server/templates/batch/header.html b/NIPTool/server/templates/batch/header.html index cd8ff4a9..41c8b182 100644 --- a/NIPTool/server/templates/batch/header.html +++ b/NIPTool/server/templates/batch/header.html @@ -2,71 +2,99 @@ {% block content %} -
-
-

Batch: {{batch._id}}

-
-
-
-
Sequenced: {{batch.SequencingDate}}
-
-

- Report - Report with coverage -

-
-
- {% if seq_warnings %} -
-
-
-
-
-
Batch Warnings
-
-
-
- - - - - - - - - - - {% for s in seq_warnings %} - - - - - - - {% endfor %} - -
Sample idMissing dataQC WarningQC Failure
{{ s }}{{seq_warnings[s]['missing_data'] }}{{ seq_warnings[s]['QC_warn'] }}{{ seq_warnings[s]['QC_fail'] }}
+
+
+

Batch: {{ batch._id }}

+
+
+
+
Sequenced: {{ batch.SequencingDate }}
+ +
+ {% if seq_warnings %} +
+
+
+
+
+
Batch Warnings
+
+
+
+ + + + + + + + + + + {% for s in seq_warnings %} + + + + + + + {% endfor %} + +
Sample idMissing dataQC WarningQC Failure
{{ s }}{{ seq_warnings[s]['missing_data'] }}{{ seq_warnings[s]['QC_warn'] }}{{ seq_warnings[s]['QC_fail'] }}
+
+
-
-
- {% endif %} -
- + {% endif %} +
+ {% endblock %} diff --git a/NIPTool/server/templates/batch/plots/NCV.html b/NIPTool/server/templates/batch/plots/NCV.html index 0d8d3b22..58a6460c 100644 --- a/NIPTool/server/templates/batch/plots/NCV.html +++ b/NIPTool/server/templates/batch/plots/NCV.html @@ -1,84 +1,107 @@ {% block scripts %} - + Plotly.newPlot('NCV{{chrom}}_plot', data, layout); + }); + {% endfor %} + - + - + {% endblock %} \ No newline at end of file diff --git a/NIPTool/server/templates/batch/report.html b/NIPTool/server/templates/batch/report.html index fa8a7f02..3d5f912b 100644 --- a/NIPTool/server/templates/batch/report.html +++ b/NIPTool/server/templates/batch/report.html @@ -54,7 +54,9 @@

NIPT Results
Batch:
+ {% if coverage=="1" %}
+ {% endif %}