From cb4a9ca6a287e5ef6ef0aa4f9b8cf7317f90f719 Mon Sep 17 00:00:00 2001 From: mayabrandi Date: Wed, 11 Nov 2020 13:44:35 +0100 Subject: [PATCH 1/3] boxplots. Showing only samples with warnings on test chromosomes --- NIPTool/server/templates/batch/coverage.html | 22 +++++-- NIPTool/server/templates/batches.html | 4 +- NIPTool/server/utils.py | 66 +++++++++++++------- NIPTool/server/views.py | 8 ++- 4 files changed, 69 insertions(+), 31 deletions(-) diff --git a/NIPTool/server/templates/batch/coverage.html b/NIPTool/server/templates/batch/coverage.html index 98b799f3..b9b6ecbb 100644 --- a/NIPTool/server/templates/batch/coverage.html +++ b/NIPTool/server/templates/batch/coverage.html @@ -37,15 +37,27 @@ linewidth: 5, title : 'Coverage Ratio' },}; - {% for samp, samp_data in data.items() %} - var trace1 = { + {% for samp, samp_data in scatter_data.items() %} + var scatter_plot = { name: {{samp|tojson}}, - y: {{samp_data}}, - x: {{x_axis}}, + y: {{samp_data.y}}, + x: {{samp_data.x}}, mode: 'markers', text: {{samp|tojson}}, type: 'scatter'}; - data.push(trace1); + data.push(scatter_plot); + {% endfor %} + {% for chr, box in box_data.items() %} + var box_plot = { + name: {{chr}}, + y: {{box}}, + x: {{chr}}, + showlegend: false, + boxpoints: false, + text: {{chr}}, + marker:{color: '#ccccb3'}, + type: 'box'}; + data.push(box_plot); {% endfor %} Plotly.newPlot('cov_plot', data, layout); diff --git a/NIPTool/server/templates/batches.html b/NIPTool/server/templates/batches.html index 971488a6..063e5482 100644 --- a/NIPTool/server/templates/batches.html +++ b/NIPTool/server/templates/batches.html @@ -31,10 +31,10 @@

NIPT Batches

{% for batch in batches %} {{batch._id if batch._id else "No Batch Name"}} + href={{ url_for('server.batch', batch_id=batch._id) }}>{{batch.SampleProject if batch.SampleProject else "No Batch Name"}} {{batch.date }} - {{ batch.flowcell}} + {{ batch._id}} {% endfor %} diff --git a/NIPTool/server/utils.py b/NIPTool/server/utils.py index 074d9f11..4eb213b9 100644 --- a/NIPTool/server/utils.py +++ b/NIPTool/server/utils.py @@ -7,6 +7,8 @@ def get_sample_info(sample): """Sample info for sample table in batch view.""" + sample_warnings = get_sample_warnings(sample) + sample_info_keys = ["Zscore_13", "Zscore_18", "Zscore_21", "CNVSegment", "FF_Formatted", "FFX", "FFY", "Zscore_X"] for key in sample_info_keys: val = sample.get(key) @@ -15,25 +17,16 @@ def get_sample_info(sample): else: sample[key] = "" - z_score_13 = sample.get("Zscore_13") - z_score_18 = sample.get("Zscore_18") - z_score_21 = sample.get("Zscore_21") - CNVSegment = sample.get("CNVSegment") - FF = sample.get("FF_Formatted") - FFX = sample.get("FFX") - FFY = sample.get("FFY") - return { "sample_id": sample.get("_id"), - "FF": {"value": FF, "warn": _get_ff_warning(FF),}, - "CNVSegment": {"value": CNVSegment, "warn": "default",}, - "FFX": {"value": FFX, "warn": "default",}, - "FFY": {"value": FFY, "warn": "default",}, - "Zscore_13": {"value": z_score_13, "warn": _get_tris_warning(z_score_13, FF),}, - "Zscore_18": {"value": z_score_18, "warn": _get_tris_warning(z_score_18, FF),}, - "Zscore_21": {"value": z_score_21, "warn": _get_tris_warning(z_score_21, FF),}, + "FF": {"value": sample.get("FF_Formatted"), "warn": sample_warnings.get("FF_Formatted")}, + "CNVSegment": {"value": sample.get("CNVSegment"), "warn": "default",}, + "FFX": {"value": sample.get("FFX"), "warn": "default",}, + "FFY": {"value": sample.get("FFY"), "warn": "default",}, + "Zscore_13": {"value": sample.get("Zscore_13"), "warn": sample_warnings.get("Zscore_13")}, + "Zscore_18": {"value": sample.get("Zscore_18"), "warn": sample_warnings.get("Zscore_18")}, + "Zscore_21": {"value": sample.get("Zscore_21"), "warn": sample_warnings.get("Zscore_21")}, "Zscore_X": {"value": sample.get("Zscore_X")}, - #'Warning': 'value': _get_warnings(sample), "Status": _get_status(sample), "Include": sample.get("include"), "Comment": sample.get("comment", ""), @@ -66,10 +59,21 @@ def _get_ff_warning(fetal_fraction): else: return "default" +def get_sample_warnings(sample): + sample_warnings = {} + fetal_fraction = sample.get('FF_Formatted') + sample_warnings["FF_Formatted"] = _get_ff_warning(fetal_fraction) + for key in ["Zscore_13", "Zscore_18", "Zscore_21"]: + z_score = sample.get(key) + sample_warnings[key] = _get_tris_warning(z_score, fetal_fraction) + + return sample_warnings + + def _get_tris_warning(z_score: float, fetal_fraction): """Get automated trisomi warning, based on preset Zscore thresholds""" - if fetal_fraction is "": + if fetal_fraction is None or z_score is None: return "default" if fetal_fraction <= 5: @@ -89,15 +93,35 @@ def _get_tris_warning(z_score: float, fetal_fraction): return warn -def get_data_for_coverage_plot(samples): +def get_scatter_data_for_coverage_plot(samples): """Coverage Ratio data for Coverage Plot.""" data = {} for sample in samples: + warnings = get_sample_warnings(sample) + warnings.pop('FF_Formatted') + if set(warnings.values()) == {'default'}: + continue sample_id = sample["_id"] - data[sample_id] = [] - for i in range(1, 23): - data[sample_id].append(sample.get(f"Chr{str(i)}_Ratio", 0)) + data[sample_id] = {"x": [], "y": []} + for chr in range(1, 23): + ratio = sample.get(f"Chr{str(chr)}_Ratio") + if ratio is None: + continue + data[sample_id]["y"].append(ratio) + data[sample_id]["x"].append(chr) + return data + +def get_box_data_for_coverage_plot(samples): + """Coverage Ratio data for Coverage Plot.""" + data = {} + for chr in range(1, 23): + data[chr] = [] + for sample in samples: + ratio = sample.get(f"Chr{str(chr)}_Ratio") + if ratio is None: + continue + data[chr].append(ratio) return data diff --git a/NIPTool/server/views.py b/NIPTool/server/views.py index 25cb5fd0..c427a2e2 100644 --- a/NIPTool/server/views.py +++ b/NIPTool/server/views.py @@ -163,13 +163,15 @@ def fetal_fraction(batch_id): def coverage(batch_id): """Batch view with coverage plot""" batch = app.adapter.batch(batch_id) - samples = app.adapter.batch_samples(batch_id) - data = get_data_for_coverage_plot(samples) + samples = list(app.adapter.batch_samples(batch_id)) + scatter_data = get_scatter_data_for_coverage_plot(samples) + box_data = get_box_data_for_coverage_plot(samples) return render_template( "batch/coverage.html", batch=batch, x_axis=list(range(1, 23)), - data=data, + scatter_data=scatter_data, + box_data = box_data, page_id="batches_cov", ) From dc153d7ad285e073d186809abe18aaaee75dca0d Mon Sep 17 00:00:00 2001 From: mayabrandi Date: Fri, 13 Nov 2020 13:51:24 +0100 Subject: [PATCH 2/3] Flowcell column --- NIPTool/server/templates/batches.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NIPTool/server/templates/batches.html b/NIPTool/server/templates/batches.html index 063e5482..31aeee6d 100644 --- a/NIPTool/server/templates/batches.html +++ b/NIPTool/server/templates/batches.html @@ -31,10 +31,10 @@

NIPT Batches

{% for batch in batches %} {{batch.SampleProject if batch.SampleProject else "No Batch Name"}} + href={{ url_for('server.batch', batch_id=batch._id) }}>{{batch._id if batch._id else "No Batch Name"}} {{batch.date }} - {{ batch._id}} + {{ batch.Flowcell}} {% endfor %} From c51142e20782e8fdc0effe6dc45d5a74b56c3e2f Mon Sep 17 00:00:00 2001 From: mayabrandi Date: Mon, 16 Nov 2020 09:06:18 +0100 Subject: [PATCH 3/3] tests --- tests/build/test_build_batch.py | 74 ------------------------ tests/build/test_build_document.py | 31 ++++++++++ tests/build/test_build_sample.py | 91 ------------------------------ 3 files changed, 31 insertions(+), 165 deletions(-) delete mode 100644 tests/build/test_build_batch.py create mode 100644 tests/build/test_build_document.py delete mode 100644 tests/build/test_build_sample.py diff --git a/tests/build/test_build_batch.py b/tests/build/test_build_batch.py deleted file mode 100644 index 3ac41a06..00000000 --- a/tests/build/test_build_batch.py +++ /dev/null @@ -1,74 +0,0 @@ -from NIPTool.build.document import build_batch -from NIPTool.models.constants import BATCH_KEYS -import pytest - - -def test_build_batch(): - # GIVEN a batch_data with requiered key 'SampleProject' - batch_data = { - "Median_18": 1.01950547134618, - "SampleProject": 201862, - "Stdev_13": 0.009517510060085, - } - - # WHEN building a mongo batch - mongo_batch = build_batch(batch_data=batch_data) - - # THEN the mongo_batch has a key "_id" with the value of "SampleProject" - assert mongo_batch == { - "_id": "201862", - "Median_18": 1.01950547134618, - "Stdev_13": 0.009517510060085, - } - - -def test_build_batch_wrong_keys(): - # GIVEN a batch_data with not accepted keys: key1 key2 key3 - batch_data = { - "SampleProject": 201862, - "key1": " ", - "key2": 201862, - "key3": -10.1836097044367, - } - - # WHEN building a mongo batch - mongo_batch = build_batch(batch_data=batch_data) - - # THEN the unaccepted keys will not be part of the mongo_batch" - assert mongo_batch == {"_id": "201862"} - - -@pytest.mark.parametrize("batch_key", BATCH_KEYS) -def test_build_batch_str_values(batch_key): - # GIVEN a batch_data dict with requiered key 'SampleProject' and a batch_key with some str value - batch_data = {"SampleProject": 201862, batch_key: "Value"} - - # WHEN building a mongo batch - mongo_batch = build_batch(batch_data=batch_data) - - # THEN the the batch_key will be loaded into the mongo_batch dict - assert mongo_batch == {"_id": "201862", batch_key: "Value"} - - -@pytest.mark.parametrize("batch_key", BATCH_KEYS) -def test_build_batch_zero_values(batch_key): - # GIVEN a batch_data dict with requiered key 'SampleProject' and a batch_key with value 0 - batch_data = {"SampleProject": 201862, batch_key: 0} - - # WHEN building a mongo batch - mongo_batch = build_batch(batch_data=batch_data) - - # THEN the the batch_key will be loaded into the mongo_batch dict - assert mongo_batch == {"_id": "201862", batch_key: 0} - - -@pytest.mark.parametrize("batch_key", BATCH_KEYS) -def test_build_batch_empty_strings(batch_key): - # GIVEN a batch_data dict with requiered key 'SampleProject' and a batch_key with empty string as value - batch_data = {"SampleProject": 201862, batch_key: " "} - - # WHEN building a mongo batch - mongo_batch = build_batch(batch_data=batch_data) - - # THEN the the batch_key will not be loaded into the mongo_batch dict - assert mongo_batch == {"_id": "201862"} \ No newline at end of file diff --git a/tests/build/test_build_document.py b/tests/build/test_build_document.py new file mode 100644 index 00000000..b8d0d17d --- /dev/null +++ b/tests/build/test_build_document.py @@ -0,0 +1,31 @@ +from NIPTool.build.document import build_batch, build_sample +from NIPTool.models.constants import BATCH_KEYS, SAMPLE_KEYS +import pytest + + +@pytest.mark.parametrize("sample_key", SAMPLE_KEYS) +@pytest.mark.parametrize("value", [124, 2.08, "randomstring"]) +def test_build_sample(sample_key, value): + # GIVEN a sample_data dict with requiered key 'SampleID' and a sample_key with some value and a non sample_key + + sample_data = {"SampleID": "someid", sample_key: value, "other_key": 8983} + + # WHEN building a mongo sample + mongo_sample = build_sample(sample_data=sample_data) + + # THEN the SampleID will be _id and the non sample key will be removed + assert mongo_sample == {"_id": "someid", sample_key: value} + + +@pytest.mark.parametrize("batch_key", BATCH_KEYS) +@pytest.mark.parametrize("value", [124, 2.08, "randomstring"]) +def test_build_batch(batch_key, value): + # GIVEN a batch_data dict a batch_key with some value and a non batch_key + + batch_data = {batch_key: value, "other_key": 8983} + + # WHEN building a mongo batch + mongo_batch = build_batch(batch_data=batch_data) + + # THEN the non batch_key will be removed + assert mongo_batch == {batch_key: value} \ No newline at end of file diff --git a/tests/build/test_build_sample.py b/tests/build/test_build_sample.py deleted file mode 100644 index 4eda2bc0..00000000 --- a/tests/build/test_build_sample.py +++ /dev/null @@ -1,91 +0,0 @@ -from NIPTool.build.document import build_sample -from NIPTool.models.constants import SAMPLE_KEYS -import pytest - - -def test_build_sample(): - # GIVEN a sample_data with requiered key 'SampleID' - sample_data = { - "SampleID": "2020-07452-02", - "Description": " ", - "Zscore_13": -10.1836097044367, - } - - # WHEN building a mongo sample - mongo_sample = build_sample(sample_data=sample_data) - - # THEN the mongo_sample has a key "_id" with the value of "SampleID" - assert mongo_sample == { - "_id": "2020-07452-02", - "Zscore_13": -10.1836097044367, - } - - -def test_build_sample_SampleProject_key(): - # GIVEN a sample_data with requiered key 'SampleID' and 'SampleProject' in int format - sample_data = { - "SampleID": "2020-07452-02", - "SampleProject": 201862, - } - - # WHEN building a mongo sample - mongo_sample = build_sample(sample_data=sample_data) - - # THEN the value of "SampleProject" is in str format - assert mongo_sample == { - "_id": "2020-07452-02", - "SampleProject": "201862", - } - - -def test_build_sample_wrong_keys(): - # GIVEN a sample_data with not accepted keys: key1 key2 key3 - sample_data = { - "SampleID": "2020-07452-02", - "key1": " ", - "key2": 201862, - "key3": -10.1836097044367, - } - - # WHEN building a mongo sample - mongo_sample = build_sample(sample_data=sample_data) - - # THEN the unaccepted keys will not be part of the mongo_sample" - assert mongo_sample == {"_id": "2020-07452-02"} - - -@pytest.mark.parametrize("sample_key", SAMPLE_KEYS) -def test_build_sample_str_values(sample_key): - # GIVEN a sample_data dict with requiered key 'SampleID' and a sample_key with some str value - sample_data = {"SampleID": "2020-07452-02", sample_key: "Value"} - - # WHEN building a mongo sample - mongo_sample = build_sample(sample_data=sample_data) - - # THEN the the sample_key will be loaded into the mongo_sample dict - if sample_key == "SampleID": - assert mongo_sample == {"_id": "2020-07452-02", sample_key: "Value"} - - -@pytest.mark.parametrize("sample_key", SAMPLE_KEYS) -def test_build_sample_empty_strings(sample_key): - # GIVEN a sample_data dict with requiered key 'SampleID' and a sample_key with empty string as value - sample_data = {"SampleID": "2020-07452-02", sample_key: " "} - - # WHEN building a mongo sample - mongo_sample = build_sample(sample_data=sample_data) - - # THEN the the sample_key will not be loaded into the mongo_sample dict - assert mongo_sample == {"_id": "2020-07452-02"} - - -@pytest.mark.parametrize("sample_key", SAMPLE_KEYS) -def test_build_sample_zero_values(sample_key): - # GIVEN a sample_data dict with requiered key 'SampleID' and a sample_key with value 0 - sample_data = {"SampleID": "2020-07452-02", sample_key: 0} - - # WHEN building a mongo sample - mongo_sample = build_sample(sample_data=sample_data) - - # THEN the the sample_key will be loaded into the mongo_sample dict - assert mongo_sample == {"_id": "2020-07452-02", sample_key: 0}