Merge 1451ba5 into cd2b8b4

NIPTool/build/document.py

@@ -23,9 +23,16 @@ def build_sample(sample_data: dict) -> dict:
     return sample
 
 
-def build_batch(batch_data: dict) -> dict:
+def build_batch(batch_data: dict, request_data: dict) -> dict:
     """Builds a document for the batch collection"""
 
     batch = build_document(batch_data, BATCH_KEYS)
+    batch["_id"] = request_data['project_name']
+    batch["fluffy_result_file"] = request_data['result_file']
+
+    if request_data.get('multiqc_report'):
+        batch["multiqc_report"] = request_data['multiqc_report']
+    if request_data.get('segmental_calls'):
+        batch["segmental_calls"] = request_data['segmental_calls']
 
     return batch

NIPTool/commands/base.py

@@ -2,7 +2,6 @@
 import logging
 
 import click
-import coloredlogs
 
 from flask.cli import FlaskGroup, with_appcontext
 from flask import current_app
@@ -12,8 +11,6 @@
 
 # Get version and doc decorator
 from NIPTool import __version__
-from NIPTool.tools.cli_utils import add_doc as doc
-from .load import load
 
 LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
 LOG = logging.getLogger(__name__)
@@ -53,4 +50,3 @@ def name():
 
 cli.add_command(test)
 cli.add_command(name)
-cli.add_command(load)

NIPTool/load/batch.py

@@ -1,54 +1,61 @@
 import logging
 from NIPTool.build.document import build_sample, build_batch
-from NIPTool.parse.batch import parse_batch_file
 from NIPTool.models.validation import requiered_fields
+from NIPTool.exeptions import MissingResultsError
+from pathlib import Path
+
+import json
 
 
 LOG = logging.getLogger(__name__)
 
 
 def check_requiered_fields(document):
-    """Check if document keys contain requiered fields"""
+    """Check if document keys contain required fields"""
 
     if not set(requiered_fields).issubset(set(document.keys())):
         LOG.info(f"Could not add document {document}. Requiered fields missing.")
         return False
     return True
 
 
-def load_result_file(adapter, batch_data: list, project_name: str) -> None:
-    """Function to load data from fluffy result file. 
-    Raises:
-        MissingResultsError: when parsing file that is empty"""
-
-    batch_data = parse_batch_file(batch_data)
+def load_batch(adapter, batch_data: dict, request_data: dict) -> None:
+    """Function to load data from fluffy result file."""
 
-    mongo_batch = build_batch(batch_data[0])
-    mongo_batch['_id'] = project_name
+    mongo_batch = build_batch(batch_data, request_data)
     adapter.add_or_update_document(mongo_batch, adapter.batch_collection)
-
+
+
+def load_samples(adapter, batch_data: list, project_name: str) -> None:
+    """Function to load data from fluffy result file."""
 
     for sample in batch_data:
         if not check_requiered_fields(sample):
             continue
         mongo_sample = build_sample(sample)
-        mongo_sample['SampleProject'] = project_name
+        mongo_sample["SampleProject"] = project_name
         adapter.add_or_update_document(mongo_sample, adapter.sample_collection)
 
-    if not check_requiered_fields(batch_data[0]):
-        return
-
 
-def load_concentrastions(adapter, concentrations: dict) -> None:
+def load_concentrations(adapter, concentrations_file: str) -> None:
     """Function to load concentrations to samples in the database."""
 
+    file = Path(concentrations_file)
+
+    if not file.exists():
+        raise MissingResultsError("Concentrations file missing.")
+
+    with open(file) as data_file:
+        concentrations = json.load(data_file)
+
     for sample, concentration in concentrations.items():
         mongo_sample = adapter.sample(sample)
         if not mongo_sample:
-            LOG.warning(f"Trying to add concentration to sample {sample} but it doesnt exist in the databse.")
+            LOG.warning(
+                f"Trying to add concentration to sample {sample} but it doesnt exist in the database."
+            )
             return
         mongo_sample["concentration"] = concentration
 
         adapter.add_or_update_document(mongo_sample, adapter.sample_collection)
 
-

NIPTool/models/validation.py

@@ -89,6 +89,15 @@
     "GC_Dropout",
     "AT_Dropout",
     "Bin2BinVariance",
-    
+
 ]
+req_str = {'type': 'string', 'required': True}
+nreq_str = {'type': 'string', 'required': False}
+
+batch_load_schema = {'concentrations': nreq_str,
+                     'project_name': req_str,
+                     'result_file': req_str,
+                     'multiqc_report': nreq_str,
+                     'segmental_calls': nreq_str}
 
+user_load_schema = {'email': req_str, 'name': req_str, 'role': req_str}

NIPTool/parse/batch.py

@@ -1,10 +1,10 @@
 import logging
-import pandas as pd
+import pandas
 from pathlib import Path
 
 from typing import Optional, List
 
-from NIPTool.exeptions import MissingResultsError, FileValidationError
+from NIPTool.exeptions import MissingResultsError
 from NIPTool.models.validation import (
     ints,
     floats,
@@ -15,8 +15,9 @@
 LOG = logging.getLogger(__name__)
 
 
+
 def form(val: Optional, function) -> Optional:
-    """Returning formated value or None"""
+    """Returning formatted value or None"""
 
     try:
         return function(val)
@@ -25,43 +26,42 @@ def form(val: Optional, function) -> Optional:
 
 
 def validate(key: str, val: Optional) -> Optional:
-    """Formating value according to defined models."""
+    """Formatting value according to defined models."""
 
     if val in exceptions:
-        formated_value = None
+        formatted_value = None
     elif key in ints:
-        formated_value = form(val, int)
+        formatted_value = form(val, int)
     elif key in floats:
-        formated_value = form(val, float)
+        formatted_value = form(val, float)
     elif key in strings:
-        formated_value = form(val, str)
+        formatted_value = form(val, str)
     else:
-        formated_value = None
-    return formated_value
+        formatted_value = None
+    return formatted_value
 
 
 def parse_batch_file(nipt_results_path: str) -> List[dict]:
-    """Parsing file content. Formating values. Ignoring values 
+    """Parsing file content. Formatting values. Ignoring values
     that could not be formatted according to defined models"""
 
     file = Path(nipt_results_path)
 
     if not file.exists():
         raise MissingResultsError("Results file missing.")
 
-    with open(file) as nipt_results_path:
-        df = pd.read_csv(file, na_filter=False)
-        results = df.to_dict(orient="records")
+    data_frame = pandas.read_csv(file, na_filter=False)
+    results = data_frame.to_dict(orient="records")
 
     samples = []
     for sample in results:
-        formated_results = {}
+        formatted_results = {}
         for key, val in sample.items():
-            formated_value = validate(key, val)
-            if formated_value is None:
+            formatted_value = validate(key, val)
+            if formatted_value is None:
                 LOG.info(f"invalid format of {key}.")
                 continue
-            formated_results[key] = formated_value
-        samples.append(formated_results)
+            formatted_results[key] = formatted_value
+        samples.append(formatted_results)
 
     return samples

NIPTool/server/__init__.py

@@ -1,15 +1,13 @@
-import os
 import logging
 
 from flask import Flask
 from pymongo import MongoClient
 import yaml
-from uuid import uuid4
-
 
 from NIPTool.adapter.plugin import NiptAdapter
 from NIPTool.server.login import login_bp, login_manager
 from NIPTool.server.views import server_bp
+from NIPTool.server.load import load_bp
 
 logging.basicConfig(level=logging.INFO)
 LOG = logging.getLogger(__name__)
@@ -31,22 +29,21 @@ def create_app(test=False):
 
 
 def configure_app(app, config=None):
-
     if config:
         app.config = {**app.config, **yaml.safe_load(config)}
     app.config['SECRET_KEY'] = app.config['SECRET_KEY']
     client = MongoClient(app.config['DB_URI'])
     db_name = app.config['DB_NAME']
     app.client = client
     app.db = client[db_name]
-    app.adapter = NiptAdapter(client, db_name = db_name)
+    app.adapter = NiptAdapter(client, db_name=db_name)
     app.register_blueprint(login_bp)
     app.register_blueprint(server_bp)
+    app.register_blueprint(load_bp)
     login_manager.init_app(app)
 
-    if app.config['DEBUG']==1:
+    if app.config['DEBUG'] == 1:
         from flask_debugtoolbar import DebugToolbarExtension
         toolbar = DebugToolbarExtension(app)
 
     return app
-