Merge 647a37f into 475fe21

NIPTool/build/document.py

@@ -0,0 +1,45 @@
+from NIPTool.models.constants import SAMPLE_KEYS, BATCH_KEYS
+from typing import Optional
+
+def empty_str_to_none(x: str) -> Optional[str]:
+    """Convert empty string to None"""
+
+    x = x.strip()
+    if not x:
+        return None
+    return x
+
+
+def build_document(csv_data: dict, document_keys: list) -> dict:
+    """Build a general document"""
+
+    document = {}
+    for key in document_keys:
+        value = csv_data.get(key)
+        if isinstance(value, str):
+            value = empty_str_to_none(value)
+        if value is None:
+            continue
+        document[key] = value
+
+    return document
+
+
+def build_sample(sample_data: dict) -> dict:
+    """Builds a document for the sample collection"""
+
+    sample = build_document(sample_data, SAMPLE_KEYS)
+    if sample.get("SampleProject"):
+        sample["SampleProject"] = str(sample["SampleProject"])
+    sample["_id"] = sample_data.get("SampleID")
+
+    return sample
+
+
+def build_batch(batch_data: dict) -> dict:
+    """Builds a document for the batch collection"""
+
+    batch = build_document(batch_data, BATCH_KEYS)
+    batch["_id"] = str(batch_data.get("SampleProject"))
+
+    return batch

NIPTool/load/batch.py

@@ -1,6 +1,5 @@
 import logging
-from NIPTool.build.sample import build_sample
-from NIPTool.build.batch import build_batch
+from NIPTool.build.document import build_sample, build_batch
 from NIPTool.parse.batch import parse_batch_file
 
 LOG = logging.getLogger(__name__)

NIPTool/constants/constants.py → NIPTool/models/constants.py

@@ -1,5 +1,4 @@
 SAMPLE_KEYS = [
-    "SampleID",
     "SampleType",
     "Description",
     "SampleProject",

NIPTool/parse/batch.py

@@ -2,12 +2,12 @@
 import pandas as pd
 import glob
 from NIPTool.exeptions import MissingResultsError, FileValidationError
-from NIPTool.models.nipt_results import nipt_results_schema
+from NIPTool.models.validation_schema import nipt_results_schema
 
 LOG = logging.getLogger(__name__)
 
 
-def parse_batch_file(nipt_results_path: dict) -> list:
+def parse_batch_file(nipt_results_path: str) -> list:
     if not glob.glob(nipt_results_path):
         raise MissingResultsError("Results file missing.")
 

NIPTool/server/templates/sample/sample_tris.html

@@ -23,7 +23,7 @@
 {% block scripts %}
 <script>  
     var layout = {
-        title: "Sample: {{sample.SampleID}}",
+        title: "Sample: {{sample._id}}",
         legend:{hovermode:'closest' },
         hovermode:'closest',
         annotations : [],

tests/build/test_build_batch.py

@@ -0,0 +1,74 @@
+from NIPTool.build.document import build_batch
+from NIPTool.models.constants import BATCH_KEYS
+import pytest
+
+
+def test_build_batch():
+    # GIVEN a batch_data with requiered key 'SampleProject'
+    batch_data = {
+        "Median_18": 1.01950547134618,
+        "SampleProject": 201862,
+        "Stdev_13": 0.009517510060085,
+    }
+
+    # WHEN building a mongo batch
+    mongo_batch = build_batch(batch_data=batch_data)
+
+    # THEN the mongo_batch has a key "_id" with the value of "SampleProject"
+    assert mongo_batch == {
+        "_id": "201862",
+        "Median_18": 1.01950547134618,
+        "Stdev_13": 0.009517510060085,
+    }
+
+
+def test_build_batch_wrong_keys():
+    # GIVEN a batch_data with not accepted keys: key1 key2 key3
+    batch_data = {
+        "SampleProject": 201862,
+        "key1": " ",
+        "key2": 201862,
+        "key3": -10.1836097044367,
+    }
+
+    # WHEN building a mongo batch
+    mongo_batch = build_batch(batch_data=batch_data)
+
+    # THEN the unaccepted keys will not be part of the mongo_batch"
+    assert mongo_batch == {"_id": "201862"}
+
+
+@pytest.mark.parametrize("batch_key", BATCH_KEYS)
+def test_build_batch_str_values(batch_key):
+    # GIVEN a batch_data dict with requiered key 'SampleProject' and a batch_key with some str value
+    batch_data = {"SampleProject": 201862, batch_key: "Value"}
+
+    # WHEN building a mongo batch
+    mongo_batch = build_batch(batch_data=batch_data)
+
+    # THEN the the batch_key will be loaded into the mongo_batch dict
+    assert mongo_batch == {"_id": "201862", batch_key: "Value"}
+
+
+@pytest.mark.parametrize("batch_key", BATCH_KEYS)
+def test_build_batch_zero_values(batch_key):
+    # GIVEN a batch_data dict with requiered key 'SampleProject' and a batch_key with value 0
+    batch_data = {"SampleProject": 201862, batch_key: 0}
+
+    # WHEN building a mongo batch
+    mongo_batch = build_batch(batch_data=batch_data)
+
+    # THEN the the batch_key will be loaded into the mongo_batch dict
+    assert mongo_batch == {"_id": "201862", batch_key: 0}
+
+
+@pytest.mark.parametrize("batch_key", BATCH_KEYS)
+def test_build_batch_empty_strings(batch_key):
+    # GIVEN a batch_data dict with requiered key 'SampleProject' and a batch_key with empty string as value
+    batch_data = {"SampleProject": 201862, batch_key: " "}
+
+    # WHEN building a mongo batch
+    mongo_batch = build_batch(batch_data=batch_data)
+
+    # THEN the the batch_key will not be loaded into the mongo_batch dict
+    assert mongo_batch == {"_id": "201862"}

tests/build/test_build_sample.py

@@ -1,24 +1,91 @@
-from NIPTool.build.sample import build_sample	
-import pytest	
+from NIPTool.build.document import build_sample
+from NIPTool.models.constants import SAMPLE_KEYS
+import pytest
 
 
-def test_build_sample():	
-   # GIVEN a sample_data with requiered key 'SampleID'	
-   sample_data = {"SampleID": "2020-07452-02",
-                  "SampleType": " ",
-                  "Description": " ",
-                  "SampleProject": 201862,
-                  "Zscore_13": -10.1836097044367}	
+def test_build_sample():
+    # GIVEN a sample_data with requiered key 'SampleID'
+    sample_data = {
+        "SampleID": "2020-07452-02",
+        "Description": " ",
+        "Zscore_13": -10.1836097044367,
+    }
 
-   # WHEN building a mongo application tag	
-   mongo_application_tag = build_sample(sample_data = sample_data)	
+    # WHEN building a mongo sample
+    mongo_sample = build_sample(sample_data=sample_data)
 
-   # THEN assert mongo_application_tag is 
-   # {"_id": "2020-07452-02","SampleID": "2020-07452-02",
-   # "SampleProject": "201862","Zscore_13": -10.1836097044367} 
+    # THEN the mongo_sample has a key "_id" with the value of "SampleID"
+    assert mongo_sample == {
+        "_id": "2020-07452-02",
+        "Zscore_13": -10.1836097044367,
+    }
 
-   assert mongo_application_tag ==  {"_id": "2020-07452-02", 
-                                     "SampleID": "2020-07452-02",
-                                     "SampleProject": "201862",
-                                     "Zscore_13": -10.1836097044367}
 
+def test_build_sample_SampleProject_key():
+    # GIVEN a sample_data with requiered key 'SampleID' and 'SampleProject' in int format
+    sample_data = {
+        "SampleID": "2020-07452-02",
+        "SampleProject": 201862,
+    }
+
+    # WHEN building a mongo sample
+    mongo_sample = build_sample(sample_data=sample_data)
+
+    # THEN the value of "SampleProject" is in str format
+    assert mongo_sample == {
+        "_id": "2020-07452-02",
+        "SampleProject": "201862",
+    }
+
+
+def test_build_sample_wrong_keys():
+    # GIVEN a sample_data with not accepted keys: key1 key2 key3
+    sample_data = {
+        "SampleID": "2020-07452-02",
+        "key1": " ",
+        "key2": 201862,
+        "key3": -10.1836097044367,
+    }
+
+    # WHEN building a mongo sample
+    mongo_sample = build_sample(sample_data=sample_data)
+
+    # THEN the unaccepted keys will not be part of the mongo_sample"
+    assert mongo_sample == {"_id": "2020-07452-02"}
+
+
+@pytest.mark.parametrize("sample_key", SAMPLE_KEYS)
+def test_build_sample_str_values(sample_key):
+    # GIVEN a sample_data dict with requiered key 'SampleID' and a sample_key with some str value
+    sample_data = {"SampleID": "2020-07452-02", sample_key: "Value"}
+
+    # WHEN building a mongo sample
+    mongo_sample = build_sample(sample_data=sample_data)
+
+    # THEN the the sample_key will be loaded into the mongo_sample dict
+    if sample_key == "SampleID":
+        assert mongo_sample == {"_id": "2020-07452-02", sample_key: "Value"}
+
+
+@pytest.mark.parametrize("sample_key", SAMPLE_KEYS)
+def test_build_sample_empty_strings(sample_key):
+    # GIVEN a sample_data dict with requiered key 'SampleID' and a sample_key with empty string as value
+    sample_data = {"SampleID": "2020-07452-02", sample_key: " "}
+
+    # WHEN building a mongo sample
+    mongo_sample = build_sample(sample_data=sample_data)
+
+    # THEN the the sample_key will not be loaded into the mongo_sample dict
+    assert mongo_sample == {"_id": "2020-07452-02"}
+
+
+@pytest.mark.parametrize("sample_key", SAMPLE_KEYS)
+def test_build_sample_zero_values(sample_key):
+    # GIVEN a sample_data dict with requiered key 'SampleID' and a sample_key with value 0
+    sample_data = {"SampleID": "2020-07452-02", sample_key: 0}
+
+    # WHEN building a mongo sample
+    mongo_sample = build_sample(sample_data=sample_data)
+
+    # THEN the the sample_key will be loaded into the mongo_sample dict
+    assert mongo_sample == {"_id": "2020-07452-02", sample_key: 0}

tests/commands/load/test_load_batch.py

@@ -0,0 +1,56 @@
+from NIPTool.commands.load.batch import batch
+from NIPTool.server import create_app
+from NIPTool.commands.base import cli
+from NIPTool.adapter.plugin import NiptAdapter
+
+
+app = create_app(test=True)
+
+
+def test_batch_valid_file(database, valid_csv):
+    app.db = database
+    app.adapter = NiptAdapter(database.client, db_name=database.name)
+
+    # GIVEN a valid csv file with three samples
+
+    # WHEN loading the batch file with correct foramted input string
+    runner = app.test_cli_runner()
+    result = runner.invoke(cli, ["load", "batch", "-b", valid_csv])
+
+    # THEN assert the new apptags should be added to the colleciton
+    assert app.adapter.sample_collection.estimated_document_count() == 3
+    assert app.adapter.batch_collection.estimated_document_count() == 1
+
+
+def test_batch_invalid_file(database, invalid_csv):
+    app.db = database
+    app.adapter = NiptAdapter(database.client, db_name=database.name)
+
+    # GIVEN a invalid csv file
+
+    # WHEN loading the batch file with correct foramted input string
+    runner = app.test_cli_runner()
+    result = runner.invoke(cli, ["load", "batch", "-b", invalid_csv])
+
+    # THEN assert nothing added to sample or batch collections
+    # THEN assert Badly formated csv! Can not load. Exiting.
+    assert app.adapter.sample_collection.estimated_document_count() == 0
+    assert app.adapter.batch_collection.estimated_document_count() == 0
+    assert result.exit_code == 1
+
+
+def test_batch_no_file(database):
+    app.db = database
+    app.adapter = NiptAdapter(database.client, db_name=database.name)
+
+    # GIVEN a invalid csv file
+
+    # WHEN loading the batch file with correct foramted input string
+    runner = app.test_cli_runner()
+    result = runner.invoke(cli, ["load", "batch", "-b", "wrong/path"])
+
+    # THEN assert nothing added to sample or batch collections
+    # THEN assert Badly formated csv! Can not load. Exiting.
+    assert app.adapter.sample_collection.estimated_document_count() == 0
+    assert app.adapter.batch_collection.estimated_document_count() == 0
+    assert result.exit_code == 1