diff --git a/NIPTool/build/document.py b/NIPTool/build/document.py index fb54d138..11f63c61 100644 --- a/NIPTool/build/document.py +++ b/NIPTool/build/document.py @@ -23,9 +23,16 @@ def build_sample(sample_data: dict) -> dict: return sample -def build_batch(batch_data: dict) -> dict: +def build_batch(batch_data: dict, request_data: dict) -> dict: """Builds a document for the batch collection""" batch = build_document(batch_data, BATCH_KEYS) + batch["_id"] = request_data['project_name'] + batch["fluffy_result_file"] = request_data['result_file'] + + if request_data.get('multiqc_report'): + batch["multiqc_report"] = request_data['multiqc_report'] + if request_data.get('segmental_calls'): + batch["segmental_calls"] = request_data['segmental_calls'] return batch diff --git a/NIPTool/commands/base.py b/NIPTool/commands/base.py index 94d5db8f..a271a6ce 100644 --- a/NIPTool/commands/base.py +++ b/NIPTool/commands/base.py @@ -2,7 +2,6 @@ import logging import click -import coloredlogs from flask.cli import FlaskGroup, with_appcontext from flask import current_app @@ -12,8 +11,6 @@ # Get version and doc decorator from NIPTool import __version__ -from NIPTool.tools.cli_utils import add_doc as doc -from .load import load LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] LOG = logging.getLogger(__name__) @@ -53,4 +50,3 @@ def name(): cli.add_command(test) cli.add_command(name) -cli.add_command(load) diff --git a/NIPTool/commands/load/__init__.py b/NIPTool/commands/load/__init__.py deleted file mode 100644 index 8ef2f5a9..00000000 --- a/NIPTool/commands/load/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base import load diff --git a/NIPTool/commands/load/base.py b/NIPTool/commands/load/base.py deleted file mode 100755 index 89f24d41..00000000 --- a/NIPTool/commands/load/base.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -import click - -# commands -from NIPTool.commands.load.batch import batch as load_batch_cmd -from NIPTool.commands.load.user import user as load_user_cmd - -# Get version -from NIPTool import __version__ - - -@click.group() -@click.version_option(version=__version__) -def load(): - """Main entry point of load commands""" - pass - - -load.add_command(load_batch_cmd) -load.add_command(load_user_cmd) diff --git a/NIPTool/commands/load/batch.py b/NIPTool/commands/load/batch.py deleted file mode 100644 index 00e0ed69..00000000 --- a/NIPTool/commands/load/batch.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging -import click -from NIPTool.load.batch import load_result_file, load_concentrastions -from flask.cli import with_appcontext, current_app -from datetime import date, timedelta -from NIPTool.exeptions import NIPToolError -import json -from pathlib import Path -from NIPTool.exeptions import MissingResultsError - - -LOG = logging.getLogger(__name__) - - -@click.command("batch", short_help="load batch into db.") -@click.option("-b", "--load-config", help="path to batch load config") -@with_appcontext -def batch(load_config: dict) -> None: - """Read and load data for one batch. - - Args: load_config - dict with keys: - "concentrations" - "result_file" - "project_name" - """ - - file = Path(load_config) - - if not file.exists(): - raise MissingResultsError("Results file missing.") - - with open(file) as data_file: - config_data = json.load(data_file) - - try: - load_result_file( - current_app.adapter, config_data["result_file"], config_data["project_name"] - ) - load_concentrastions(current_app.adapter, config_data["concentrations"]) - except NIPToolError as e: - LOG.error(e.message) - raise click.Abort() diff --git a/NIPTool/commands/load/user.py b/NIPTool/commands/load/user.py deleted file mode 100644 index e5443404..00000000 --- a/NIPTool/commands/load/user.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging -import click -from NIPTool.load.user import load_user -from flask.cli import with_appcontext, current_app -from NIPTool.exeptions import NIPToolError - - -LOG = logging.getLogger(__name__) - - -@click.command("user", short_help="load a user into db.") -@click.option("-n", "--name", help="User name") -@click.option("-r", "--role", help="User role") -@click.option("-e", "--email", help="User email") -@with_appcontext -def user(name, role, email): - """Loading new user to db.""" - - try: - load_user(current_app.adapter, email, name, role) - except NIPToolError as e: - LOG.error(e.message) - raise click.Abort() diff --git a/NIPTool/load/batch.py b/NIPTool/load/batch.py index b0363e98..ced00930 100644 --- a/NIPTool/load/batch.py +++ b/NIPTool/load/batch.py @@ -1,14 +1,17 @@ import logging from NIPTool.build.document import build_sample, build_batch -from NIPTool.parse.batch import parse_batch_file from NIPTool.models.validation import requiered_fields +from NIPTool.exeptions import MissingResultsError +from pathlib import Path + +import json LOG = logging.getLogger(__name__) def check_requiered_fields(document): - """Check if document keys contain requiered fields""" + """Check if document keys contain required fields""" if not set(requiered_fields).issubset(set(document.keys())): LOG.info(f"Could not add document {document}. Requiered fields missing.") @@ -16,39 +19,43 @@ def check_requiered_fields(document): return True -def load_result_file(adapter, batch_data: list, project_name: str) -> None: - """Function to load data from fluffy result file. - Raises: - MissingResultsError: when parsing file that is empty""" - - batch_data = parse_batch_file(batch_data) +def load_batch(adapter, batch_data: dict, request_data: dict) -> None: + """Function to load data from fluffy result file.""" - mongo_batch = build_batch(batch_data[0]) - mongo_batch['_id'] = project_name + mongo_batch = build_batch(batch_data, request_data) adapter.add_or_update_document(mongo_batch, adapter.batch_collection) - + + +def load_samples(adapter, batch_data: list, project_name: str) -> None: + """Function to load data from fluffy result file.""" for sample in batch_data: if not check_requiered_fields(sample): continue mongo_sample = build_sample(sample) - mongo_sample['SampleProject'] = project_name + mongo_sample["SampleProject"] = project_name adapter.add_or_update_document(mongo_sample, adapter.sample_collection) - if not check_requiered_fields(batch_data[0]): - return - -def load_concentrastions(adapter, concentrations: dict) -> None: +def load_concentrations(adapter, concentrations_file: str) -> None: """Function to load concentrations to samples in the database.""" + file = Path(concentrations_file) + + if not file.exists(): + raise MissingResultsError("Concentrations file missing.") + + with open(file) as data_file: + concentrations = json.load(data_file) + for sample, concentration in concentrations.items(): mongo_sample = adapter.sample(sample) if not mongo_sample: - LOG.warning(f"Trying to add concentration to sample {sample} but it doesnt exist in the databse.") + LOG.warning( + f"Trying to add concentration to sample {sample} but it doesnt exist in the database." + ) return mongo_sample["concentration"] = concentration adapter.add_or_update_document(mongo_sample, adapter.sample_collection) - diff --git a/NIPTool/models/validation.py b/NIPTool/models/validation.py index 8410733d..36fc54e0 100644 --- a/NIPTool/models/validation.py +++ b/NIPTool/models/validation.py @@ -89,6 +89,15 @@ "GC_Dropout", "AT_Dropout", "Bin2BinVariance", - + ] +req_str = {'type': 'string', 'required': True} +nreq_str = {'type': 'string', 'required': False} + +batch_load_schema = {'concentrations': nreq_str, + 'project_name': req_str, + 'result_file': req_str, + 'multiqc_report': nreq_str, + 'segmental_calls': nreq_str} +user_load_schema = {'email': req_str, 'name': req_str, 'role': req_str} diff --git a/NIPTool/parse/batch.py b/NIPTool/parse/batch.py index c7add708..30747e8a 100644 --- a/NIPTool/parse/batch.py +++ b/NIPTool/parse/batch.py @@ -1,10 +1,10 @@ import logging -import pandas as pd +import pandas from pathlib import Path from typing import Optional, List -from NIPTool.exeptions import MissingResultsError, FileValidationError +from NIPTool.exeptions import MissingResultsError from NIPTool.models.validation import ( ints, floats, @@ -15,8 +15,9 @@ LOG = logging.getLogger(__name__) + def form(val: Optional, function) -> Optional: - """Returning formated value or None""" + """Returning formatted value or None""" try: return function(val) @@ -25,23 +26,23 @@ def form(val: Optional, function) -> Optional: def validate(key: str, val: Optional) -> Optional: - """Formating value according to defined models.""" + """Formatting value according to defined models.""" if val in exceptions: - formated_value = None + formatted_value = None elif key in ints: - formated_value = form(val, int) + formatted_value = form(val, int) elif key in floats: - formated_value = form(val, float) + formatted_value = form(val, float) elif key in strings: - formated_value = form(val, str) + formatted_value = form(val, str) else: - formated_value = None - return formated_value + formatted_value = None + return formatted_value def parse_batch_file(nipt_results_path: str) -> List[dict]: - """Parsing file content. Formating values. Ignoring values + """Parsing file content. Formatting values. Ignoring values that could not be formatted according to defined models""" file = Path(nipt_results_path) @@ -49,19 +50,18 @@ def parse_batch_file(nipt_results_path: str) -> List[dict]: if not file.exists(): raise MissingResultsError("Results file missing.") - with open(file) as nipt_results_path: - df = pd.read_csv(file, na_filter=False) - results = df.to_dict(orient="records") + data_frame = pandas.read_csv(file, na_filter=False) + results = data_frame.to_dict(orient="records") samples = [] for sample in results: - formated_results = {} + formatted_results = {} for key, val in sample.items(): - formated_value = validate(key, val) - if formated_value is None: + formatted_value = validate(key, val) + if formatted_value is None: LOG.info(f"invalid format of {key}.") continue - formated_results[key] = formated_value - samples.append(formated_results) + formatted_results[key] = formatted_value + samples.append(formatted_results) return samples diff --git a/NIPTool/server/__init__.py b/NIPTool/server/__init__.py index 560e5611..bf263f5f 100644 --- a/NIPTool/server/__init__.py +++ b/NIPTool/server/__init__.py @@ -1,15 +1,13 @@ -import os import logging from flask import Flask from pymongo import MongoClient import yaml -from uuid import uuid4 - from NIPTool.adapter.plugin import NiptAdapter from NIPTool.server.login import login_bp, login_manager from NIPTool.server.views import server_bp +from NIPTool.server.load import load_bp logging.basicConfig(level=logging.INFO) LOG = logging.getLogger(__name__) @@ -31,7 +29,6 @@ def create_app(test=False): def configure_app(app, config=None): - if config: app.config = {**app.config, **yaml.safe_load(config)} app.config['SECRET_KEY'] = app.config['SECRET_KEY'] @@ -39,14 +36,14 @@ def configure_app(app, config=None): db_name = app.config['DB_NAME'] app.client = client app.db = client[db_name] - app.adapter = NiptAdapter(client, db_name = db_name) + app.adapter = NiptAdapter(client, db_name=db_name) app.register_blueprint(login_bp) app.register_blueprint(server_bp) + app.register_blueprint(load_bp) login_manager.init_app(app) - if app.config['DEBUG']==1: + if app.config['DEBUG'] == 1: from flask_debugtoolbar import DebugToolbarExtension toolbar = DebugToolbarExtension(app) return app - diff --git a/NIPTool/server/load.py b/NIPTool/server/load.py new file mode 100644 index 00000000..56795aff --- /dev/null +++ b/NIPTool/server/load.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +from flask import ( + jsonify, + request, + Blueprint, + current_app) + +from NIPTool.parse.batch import parse_batch_file +from NIPTool.load.batch import load_batch, load_samples, load_concentrations +from NIPTool.load.user import load_user +from NIPTool.models.validation import user_load_schema, batch_load_schema +from cerberus import Validator + +import logging + +LOG = logging.getLogger(__name__) + +app = current_app +load_bp = Blueprint("load", __name__) + + +@load_bp.route("/batch", methods=["POST"]) +def batch(): + """Function to load batch data into the database with rest""" + + request_data = request.form + v = Validator(batch_load_schema) + if not v.validate(request_data): + message = "Incomplete batch load request" + resp = jsonify({"message": message}) + resp.status_code = 400 + return resp + + batch_data = parse_batch_file(request_data['result_file']) + load_batch(current_app.adapter, batch_data[0], request_data) + load_samples(current_app.adapter, batch_data, request_data['project_name']) + load_concentrations(current_app.adapter, request_data["concentrations"]) + + message = "Data loaded into database" + resp = jsonify({"message": message}) + resp.status_code = 200 + return resp + + +@load_bp.route("/user", methods=["POST"]) +def user(): + """Function to load user into the database with rest""" + + request_data = request.form + v = Validator(user_load_schema) + if not v.validate(request_data): + message = "Incomplete user load request" + resp = jsonify({"message": message}) + resp.status_code = 400 + return resp + + load_user(current_app.adapter, request_data["email"], request_data["name"], request_data["role"]) + + message = "Data loaded into database" + resp = jsonify({"message": message}) + resp.status_code = 200 + return resp diff --git a/NIPTool/server/login.py b/NIPTool/server/login.py index c77bd7f0..b4c71367 100644 --- a/NIPTool/server/login.py +++ b/NIPTool/server/login.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from flask import url_for, redirect, request, Blueprint, session, current_app, flash +from flask import url_for, redirect, Blueprint, session, current_app, flash from flask_login import LoginManager from authlib.integrations.flask_client import OAuth diff --git a/NIPTool/server/views.py b/NIPTool/server/views.py index f1c63159..20902603 100644 --- a/NIPTool/server/views.py +++ b/NIPTool/server/views.py @@ -16,6 +16,12 @@ from NIPTool.server.utils import * from NIPTool.server.constants import * + +import logging + +LOG = logging.getLogger(__name__) + + app = current_app server_bp = Blueprint("server", __name__) @@ -37,6 +43,7 @@ def batches(): return render_template("batches.html", batches=all_batches) + ### Batch tabs @server_bp.route("/batches//") @@ -219,7 +226,6 @@ def download(batch_id, file_id): # Statistics - @server_bp.route("/statistics") @login_required def statistics(): @@ -264,6 +270,7 @@ def statistics(): @login_required def update(): """Update the database""" + time_stamp = datetime.now().strftime("%Y/%m/%d %H:%M:%S") user = app.user if user.role != "RW": @@ -315,3 +322,4 @@ def update(): ) return redirect(request.referrer) + diff --git a/requirements.txt b/requirements.txt index af2c05c1..d95ceca8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ requests flask_ldap3_login pandas pandas-schema +cerberus + diff --git a/tests/build/test_build_document.py b/tests/build/test_build_document.py index b8d0d17d..b7a1f3ba 100644 --- a/tests/build/test_build_document.py +++ b/tests/build/test_build_document.py @@ -1,5 +1,5 @@ from NIPTool.build.document import build_batch, build_sample -from NIPTool.models.constants import BATCH_KEYS, SAMPLE_KEYS +from NIPTool.models.constants import BATCH_KEYS, SAMPLE_KEYS import pytest @@ -23,9 +23,11 @@ def test_build_batch(batch_key, value): # GIVEN a batch_data dict a batch_key with some value and a non batch_key batch_data = {batch_key: value, "other_key": 8983} - + project_name = "project_name" + file_path = "path" + request_data = {"result_file": file_path, "project_name": project_name} # WHEN building a mongo batch - mongo_batch = build_batch(batch_data=batch_data) + mongo_batch = build_batch(batch_data=batch_data, request_data=request_data) # THEN the non batch_key will be removed - assert mongo_batch == {batch_key: value} \ No newline at end of file + assert mongo_batch == {'_id': project_name, 'fluffy_result_file': file_path, batch_key: value} diff --git a/tests/commands/load/test_load_batch.py b/tests/commands/load/test_load_batch.py deleted file mode 100644 index e43b0f78..00000000 --- a/tests/commands/load/test_load_batch.py +++ /dev/null @@ -1,57 +0,0 @@ -from NIPTool.commands.load.batch import batch -from NIPTool.server import create_app -from NIPTool.commands.base import cli -from NIPTool.adapter.plugin import NiptAdapter - - -app = create_app(test=True) - - -def test_batch_valid_file(database, valid_load_config): - app.db = database - app.adapter = NiptAdapter(database.client, db_name=database.name) - - # GIVEN a valid csv file with three samples - - # WHEN loading the batch file with correct foramted input string - runner = app.test_cli_runner() - runner.invoke(cli, ["load", "batch", "-b", valid_load_config]) - - # THEN assert the samples should be added to the sample colleciton - # and the batch should be added to the batch colleciton - assert app.adapter.sample_collection.estimated_document_count() == 3 - assert app.adapter.batch_collection.estimated_document_count() == 1 - - -def test_batch_invalid_file(database, invalid_load_config): - app.db = database - app.adapter = NiptAdapter(database.client, db_name=database.name) - - # GIVEN a invalid csv file - - # WHEN loading the batch file with correct foramted input string - runner = app.test_cli_runner() - result = runner.invoke(cli, ["load", "batch", "-b", invalid_load_config]) - - # THEN assert nothing added to sample or batch collections - # THEN assert Badly formated csv! Can not load. Exiting. - assert app.adapter.sample_collection.estimated_document_count() == 0 - assert app.adapter.batch_collection.estimated_document_count() == 0 - assert result.exit_code == 1 - - -def test_batch_no_file(database): - app.db = database - app.adapter = NiptAdapter(database.client, db_name=database.name) - - # GIVEN a invalid csv file - - # WHEN loading the batch file with correct foramted input string - runner = app.test_cli_runner() - result = runner.invoke(cli, ["load", "batch", "-b", "wrong/path"]) - - # THEN assert nothing added to sample or batch collections - # THEN assert Badly formated csv! Can not load. Exiting. - assert app.adapter.sample_collection.estimated_document_count() == 0 - assert app.adapter.batch_collection.estimated_document_count() == 0 - assert result.exit_code == 1 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 2fdac8e5..c38dd965 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,10 +2,30 @@ from mongomock import MongoClient from .small_helpers import SmallHelpers +from NIPTool.server import create_app, configure_app +from NIPTool.adapter.plugin import NiptAdapter +from NIPTool.server.load import load_bp DATABASE = "testdb" +@pytest.fixture(scope="function") +def database(request, pymongo_client): + """Get an adapter connected to mongo database""" + + mongo_client = pymongo_client + database = mongo_client[DATABASE] + return database + +@pytest.fixture() +def mock_app(database): + """Return a class with small helper functions""" + app = create_app(test=True) + app.db = database + app.adapter = NiptAdapter(database.client, db_name=database.name) + app.register_blueprint(load_bp) + return app + @pytest.fixture(scope="function") def pymongo_client(request): @@ -20,13 +40,7 @@ def teardown(): return mock_client -@pytest.fixture(scope="function") -def database(request, pymongo_client): - """Get an adapter connected to mongo database""" - mongo_client = pymongo_client - database = mongo_client[DATABASE] - return database @pytest.fixture(name="small_helpers") @@ -55,15 +69,26 @@ def invalid_csv(): @pytest.fixture -def valid_load_config(): +def valid_concentrations(): """Get file path to valid csv""" - return "tests/fixtures/valid_load_config.yaml" + return "tests/fixtures/valid_concentrations.yaml" @pytest.fixture -def invalid_load_config(): - """Get file path to invalid csv""" +def invalid_concentrations(): + """Get file path to invalid concentrations file""" - return "tests/fixtures/not_valid_load_config.yaml" + return "tests/fixtures/not_valid_concentrations.yaml" + +@pytest.fixture +def multiqc_report(): + """Get file path to multiqc_report""" + + return "tests/fixtures/multiqc_report.html" + +@pytest.fixture +def segmental_calls(): + """Get file path to segmental_calls""" + return "tests/fixtures/segmental_calls.bed" diff --git a/tests/fixtures/multiqc_report.html b/tests/fixtures/multiqc_report.html new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/not_valid_concentrations.yaml b/tests/fixtures/not_valid_concentrations.yaml new file mode 100644 index 00000000..ec876604 --- /dev/null +++ b/tests/fixtures/not_valid_concentrations.yaml @@ -0,0 +1 @@ +{ "2020-07547-02": 12, "2020-07649-02": 12, "2020-07455-02": 12 diff --git a/tests/fixtures/not_valid_load_config.yaml b/tests/fixtures/not_valid_load_config.yaml deleted file mode 100644 index 0adb4023..00000000 --- a/tests/fixtures/not_valid_load_config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -{ - "concentrations": - { "2020-07547-02": 12, "2020-07649-02": 12, "2020-07455-02": 12 }, - "result_file": "tests/fixtures/not_a_valid_fluffy.csv", - "project_name": "project1" diff --git a/tests/fixtures/segmental_calls.bed b/tests/fixtures/segmental_calls.bed new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/valid_concentrations.yaml b/tests/fixtures/valid_concentrations.yaml new file mode 100644 index 00000000..d36af689 --- /dev/null +++ b/tests/fixtures/valid_concentrations.yaml @@ -0,0 +1 @@ +{ "s1": 12, "s2": 12, "s3": 12 } \ No newline at end of file diff --git a/tests/fixtures/valid_load_config.yaml b/tests/fixtures/valid_load_config.yaml deleted file mode 100644 index e43a0b27..00000000 --- a/tests/fixtures/valid_load_config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{ - "concentrations": - { "s1": 12, "s2": 12, "s3": 12 }, - "result_file": "tests/fixtures/valid_fluffy.csv", - "project_name": "testprojet" -} \ No newline at end of file diff --git a/tests/server/__init__.py b/tests/server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/server/test_load.py b/tests/server/test_load.py new file mode 100644 index 00000000..1e5915d1 --- /dev/null +++ b/tests/server/test_load.py @@ -0,0 +1,108 @@ +import json + + +def test_user(mock_app): + # GIVEN the following request data: + data = dict(email='maya.papaya@something.se', name="Maya Papaya", role="RW") + + # WHEN running the user request with the data + response = mock_app.test_client().post('/user', data=data) + + # The user should be added to the database + assert mock_app.adapter.user_collection.estimated_document_count() == 1 + resp_data = json.loads(response.data) + assert resp_data["message"] == "Data loaded into database" + assert response.status_code == 200 + +def test_user_empty_data(mock_app): + # GIVEN no data + + # WHEN running the user request with empty data + response = mock_app.test_client().post('/user', data=dict()) + + # THEN assert BadRequestKeyError: 400 Bad Request + assert response.status_code == 400 + + +def test_batch_valid_files(mock_app, valid_concentrations, valid_csv, segmental_calls, multiqc_report): + # GIVEN the following request data: + # a valid csv file with three samples + # a valid concentrations file + # segmental_calls and multiqc_report files with random content, but that do exist. + + project_name = "project_name" + data = dict(multiqc_report=multiqc_report, segmental_calls=segmental_calls, result_file=valid_csv, + concentrations=valid_concentrations, project_name=project_name) + + # WHEN running the request with the data + response = mock_app.test_client().post('/batch', data=data) + + # THEN assert the samples should be added to the sample collection + # and the batch should be added to the batch collection + assert mock_app.adapter.sample_collection.estimated_document_count() == 3 + assert mock_app.adapter.batch_collection.estimated_document_count() == 1 + resp_data = json.loads(response.data) + assert resp_data["message"] == "Data loaded into database" + assert response.status_code == 200 + + +def test_batch_no_data(mock_app): + # GIVEN no data + + # WHEN running the request with empty data + response = mock_app.test_client().post('/batch', data=dict()) + + # THEN assert nothing added to sample or batch collections + assert mock_app.adapter.sample_collection.estimated_document_count() == 0 + assert mock_app.adapter.batch_collection.estimated_document_count() == 0 + + # THEN assert BadRequestKeyError: 400 Bad Request + assert response.status_code == 400 + + +def test_batch_missing_files(mock_app, valid_concentrations, valid_csv): + # GIVEN the following request data: + # a valid csv file with three samples + # a valid concentrations file + # but no segmental_calls and multiqc_report + + data = dict(result_file=valid_csv, concentrations=valid_concentrations, project_name="project_name") + + # WHEN running the request with the data + response = mock_app.test_client().post('/batch', data=data) + + # THEN assert the samples are being added + assert mock_app.adapter.sample_collection.estimated_document_count() == 3 + + # THEN assert batch is being added without error + assert mock_app.adapter.batch_collection.estimated_document_count() == 1 + resp_data = json.loads(response.data) + assert resp_data["message"] == "Data loaded into database" + assert response.status_code == 200 + + +def test_batch_invalid_file(mock_app, valid_concentrations, invalid_csv, segmental_calls, multiqc_report): + # GIVEN the following request data: + # a invalid csv file with three samples - required field SampleID has bad format + # a valid concentrations file + # segmental_calls and multiqc_report files with random content, but that do exist. + + project_name = "project_name" + data = dict(multiqc_report=multiqc_report, segmental_calls=segmental_calls, result_file=invalid_csv, + concentrations=valid_concentrations, project_name=project_name) + + # WHEN running the3 request with the data + response = mock_app.test_client().post('/batch', data=data) + + # THEN assert nothing added to sample the collection + assert mock_app.adapter.sample_collection.estimated_document_count() == 0 + + # THEN assert batch collection added with expected keys + keys = ['_id', 'multiqc_report', 'segmental_calls', 'fluffy_result_file', 'added'] + assert mock_app.adapter.batch_collection.estimated_document_count() == 1 + assert set(mock_app.adapter.batch(project_name).keys()) == set(keys) + resp_data = json.loads(response.data) + assert resp_data["message"] == "Data loaded into database" + assert response.status_code == 200 + + diff --git a/tests/server/test_utils.py b/tests/server/test_utils.py index 5a5dcbbb..9c95f09a 100644 --- a/tests/server/test_utils.py +++ b/tests/server/test_utils.py @@ -1,77 +1,61 @@ -from NIPTool.server import create_app from NIPTool.server.utils import ( get_last_batches, get_statistics_for_box_plot, get_statistics_for_scatter_plot, ) -from NIPTool.adapter.plugin import NiptAdapter -from datetime import datetime -app = create_app(test=True) - - -def test_get_last_batches(database): - app.db = database - app.adapter = NiptAdapter(database.client, db_name=database.name) - +def test_get_last_batches(mock_app): # GIVEN a database with four batch documents: batch1 = {"SequencingDate": "2022-03-10"} batch2 = {"SequencingDate": "2022-02-10"} batch3 = {"SequencingDate": "2022-02-09"} batch4 = {"SequencingDate": "2021-03-10"} - database.batch.insert_many([batch4, batch1, batch3, batch2]) + mock_app.db.batch.insert_many([batch4, batch1, batch3, batch2]) # WHEN running get_last_batches with nr=3 - results = get_last_batches(adapter=app.adapter, nr=3) + results = get_last_batches(adapter=mock_app.adapter, nr=3) # THEN the results should contain the thre batches with the latest # SequencingDate, sorted by SequencingDate assert results == [batch1, batch2, batch3] -def test_get_last_batches_to_fiew_batches_in_database(database): - app.db = database - app.adapter = NiptAdapter(database.client, db_name=database.name) - +def test_get_last_batches_to_fiew_batches_in_database(mock_app): # GIVEN a database with four batch documents: batch1 = {"SequencingDate": "2022-03-10"} batch2 = {"SequencingDate": "2022-02-10"} batch3 = {"SequencingDate": "2022-02-09"} batch4 = {"SequencingDate": "2021-03-10"} - database.batch.insert_many([batch4, batch1, batch3, batch2]) + mock_app.db.batch.insert_many([batch4, batch1, batch3, batch2]) # WHEN running get_last_batches with nr=5 - results = get_last_batches(adapter=app.adapter, nr=5) + results = get_last_batches(adapter=mock_app.adapter, nr=5) # THEN the results should contain the four batches in the # database sorted by SequencingDate assert results == [batch1, batch2, batch3, batch4] -def test_get_statistics_for_box_plot(database, small_helpers): - app.db = database - app.adapter = NiptAdapter(database.client, db_name=database.name) - +def test_get_statistics_for_box_plot(mock_app, small_helpers): # GIVEN a database with thre batches and nine samples like below batches = ["201860", "201861", "201862"] for batch_id in batches: - batch = small_helpers.batch(batch_id = batch_id) - database.batch.insert_one(batch) - for i in [1,2,3]: + batch = small_helpers.batch(batch_id=batch_id) + mock_app.db.batch.insert_one(batch) + for i in [1, 2, 3]: sample = small_helpers.sample( - sample_id = batch_id+str(i), - batch_id = batch_id, - ratio_13=i, - fetal_fraction=i) - database.sample.insert_one(sample) - + sample_id=batch_id + str(i), + batch_id=batch_id, + ratio_13=i, + fetal_fraction=i) + mock_app.db.sample.insert_one(sample) # WHEN running get_statistics_for_box_plot results = get_statistics_for_box_plot( - adapter=app.adapter, + adapter=mock_app.adapter, batches=batches, fields=["Ratio_13", "FF_Formatted"], ) @@ -85,10 +69,7 @@ def test_get_statistics_for_box_plot(database, small_helpers): } -def test_get_statistics_for_scatter_plot(database, small_helpers): - app.db = database - app.adapter = NiptAdapter(database.client, db_name=database.name) - +def test_get_statistics_for_scatter_plot(small_helpers): # GIVEN a list with three batch documents: batches = [ small_helpers.batch(batch_id="201860", stdev_13=0.02, stdev_21=0.01),