Merge 4f8d547 into 0e5c51a

NIPTool/adapter/plugin.py

@@ -59,6 +59,10 @@ def sample_aggregate(self, pipe: list):
         """Aggregates a query pipeline on the sample collection"""
         return self.sample_collection.aggregate(pipe)
 
+    def batch_aggregate(self, pipe: list):
+        """Aggregates a query pipeline on the sample collection"""
+        return self.batch_collection.aggregate(pipe)
+
     def batch_samples(self, batch_id):
         """All samples within the batch"""
         return self.sample_collection.find({"SampleProject": batch_id})

NIPTool/server/__init__.py

@@ -49,3 +49,4 @@ def configure_app(app, config=None):
         toolbar = DebugToolbarExtension(app)
 
     return app
+

NIPTool/server/templates/layout/navigation.html

@@ -12,6 +12,8 @@
             <ul class="dropdown-menu">
                 <a class="active" href="{{ url_for('server.batches') }}"><i class="fa fa-home"></i> Start</a>
                 <li class="divider"></li>
+                <a class="active" href={{ url_for('server.statistics') }}><i class="fa fa-bar-chart"></i>  QC</a>
+                <li class="divider"></li>
                 <a class="active" href={{ url_for('login.logout') }}><i class="fa fa-sign-out"></i>  Sign Out</a>
                 <li class="divider"></li>
             </ul>

NIPTool/server/templates/statistics.html

@@ -0,0 +1,133 @@
+{% extends 'layout/layout.html' %}
+{% block body %}
+{% include 'layout/navigation.html' %}
+<div class="container-fluid">
+    <div class="jumbotron vertical-center">
+        <div class="row">
+            <div class="col-lg-12">
+                <h1 class="page-header"><strong>Quality Control/QC </strong></h1>
+            </div>
+        </div>
+    </div>
+    <div class="row">
+    </div>
+    <ul class="nav nav-tabs" id="myTab">
+        {% for plot in box_plots %}
+        <li><a href="#{{plot}}" data-toggle="tab">{{plot}}</a></li>
+        {% endfor %}
+        {% for plot in scatter_plots %}
+        <li><a href="#{{plot}}" data-toggle="tab">{{plot}}</a></li>
+        {% endfor %}
+    </ul>
+    <div class="tab-content">
+        {% for plot in box_plots %}
+        <div class="tab-pane fade in active" id='{{plot}}'>
+            <div id="{{plot}}" style="min-width:1000px; height:600px; "></div>
+        </div>
+        {% endfor %}
+        {% for plot in scatter_plots %}
+        <div class="tab-pane fade in active" id='{{plot}}'>
+            <div id="{{plot}}" style="min-width:1000px; height:600px; "></div>
+        </div>
+        {% endfor %}
+    </div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+
+<script>
+
+    ////////////////// Box Plots //////////////////
+    {% for plot_name in box_plots %}
+    var XY_plot = document.getElementById('')
+    var data = []
+    var layout = {
+        title: '{{plot_name}} - ({{nr_batches}} most recent batches)',
+        hovermode: 'closest',
+        margin: { b: 100 },
+        xaxis: {
+            showline: true,
+            tickvals: {{ ticks }},
+            ticktext: {{ batch_ids | tojson }},
+            tickangle: 40,
+            zeroline: false,
+            linecolor: '#636363',
+            linewidth: 5,
+            gridcolor: '#bdbdbd'},
+        yaxis: {
+            zeroline: false,
+            showline: true,
+            showgrid: false,
+            linecolor: '#636363',
+            linewidth: 5,
+            title : "{{plot_name}}"
+            }};
+
+    {% for batch in box_stat %}
+    var box = {
+        y: {{ batch[plot_name] }},
+        x: Array({{ nr_batches }}).fill({{ loop.index }}),
+        type: 'box',
+        showlegend: false,
+        boxpoints: 'suspectedoutliers',
+        };
+    data.push(box);
+    {% endfor %}
+
+    Plotly.newPlot("{{plot_name}}", data, layout);
+    {% endfor %}
+</script>
+
+<script>
+    jQuery(function () {
+        jQuery('#myTab a:first').tab('show')
+    })
+</script>
+
+<script>
+    ////////////////// Scatter plots //////////////////
+    {% for plot_name in scatter_plots %}
+    var XY_plot = document.getElementById('')
+    var data = []
+
+    var layout = {
+            title: '{{plot_name}} - ({{nr_batches}} most recent batches)',
+            hovermode: 'closest',
+            margin: { b: 100 },
+            xaxis: {
+                showline: true,
+                tickvals: {{ ticks }},
+                ticktext: {{ batch_ids| tojson }},
+                tickangle: 40,
+                zeroline: false,
+                linecolor: '#636363',
+                linewidth: 5,
+                gridcolor: '#bdbdbd',},
+            yaxis: {
+                zeroline: false,
+                showline: true,
+                showgrid: false,
+                linecolor: '#636363',
+                linewidth: 5,
+                title : "{{plot_name}}"}
+                }
+
+
+    {% for batch_id in scatter_stat %}
+        var batch = {
+            y: [{{ scatter_stat[batch_id][plot_name] }}],
+            x: [{{ loop.index  }}],
+            type: 'scatter',
+            mode: 'markers',
+            showlegend: false,
+            name: "{{batch_id}}",
+            text: "Date Run: {{scatter_stat[batch_id]['date']}}"};                          
+        data.push(batch);
+    {% endfor %}
+
+    Plotly.newPlot("{{plot_name}}", data, layout);
+    {% endfor %}
+</script>
+
+{% endblock %}

NIPTool/server/utils.py

@@ -301,3 +301,55 @@ def get_sample_for_samp_tris_plot(sample):
         "18": {"value": sample.get("Zscore_18"), "x_axis": 2},
         "21": {"value": sample.get("Zscore_21"), "x_axis": 3},
     }
+
+
+def get_last_batches(adapter, nr: int) -> list:
+    """geting the <nr> last batches based on SequencingDate"""
+
+    batch_sort_aggregation = [{'$sort': {'SequencingDate': -1}}]
+    sorted_batches = list(adapter.batch_aggregate(batch_sort_aggregation))
+    if len(sorted_batches) > nr:
+        sorted_batches = sorted_batches[0:nr] 
+
+    return(sorted_batches)
+
+
+def get_statistics_for_scatter_plot(batches: list, fields: list)-> dict:
+    """Formating data for scatter plot"""
+
+    scatter_plot_data = {}
+    for batch in batches:
+        batch_id = batch.get('_id')
+        scatter_plot_data[batch_id] = {
+            'date': batch.get('SequencingDate')}
+        for field in fields:
+            scatter_plot_data[batch_id][field] = batch.get(field)
+
+    return scatter_plot_data
+
+
+def get_statistics_for_box_plot(adapter, batches: list, fields: list):
+    """Getting and formating data for box plot"""
+
+    match = {'$match': {'SampleProject': {'$in': batches}}}
+    lookup = {'$lookup': {
+                'from': 'batch', 
+                'localField': 'SampleProject', 
+                'foreignField': '_id', 
+                'as': 'batch'}} 
+    unwind = {'$unwind': {'path': '$batch'}}
+    group = {'$group': {'_id': {
+                            'batch': '$SampleProject', 
+                            'date': '$batch.SequencingDate'}}}
+
+    for field in fields:
+        group['$group'][field] = {'$push': f"${field}"}
+
+    pipe = [match, lookup, unwind, group]
+            #maybe add a fina sort to the pipe
+    box_plot_data = list(adapter.sample_aggregate(pipe))
+    return box_plot_data
+
+
+
+

NIPTool/server/views.py

@@ -36,6 +36,33 @@ def batches():
     return render_template("batches.html", batches=all_batches)
 
 
+@server_bp.route("/statistics")
+@login_required
+def statistics():
+    """Statistics view."""
+
+    nr_batches = 3
+    scatter_plots = ['Stdev_13', 'Stdev_18', 'Stdev_21']
+    box_plots = ['Ratio_13', 'Ratio_18', 'Ratio_21','FetalFraction', 
+                'DuplicationRate', 'MappedReads', 'GC_Dropout']
+
+    batches = get_last_batches(adapter=app.adapter, nr=nr_batches)
+    batch_ids = [batch.get('_id') for batch in batches]
+    box_stat = get_statistics_for_box_plot(adapter=app.adapter, batches=batch_ids, fields=box_plots)
+    scatter_stat = get_statistics_for_scatter_plot(batches=batches, fields=scatter_plots)
+    print(scatter_stat)
+    return render_template("statistics.html",
+        ticks = list(range(1, nr_batches+1)),
+        nr_batches = nr_batches,
+        batch_ids = batch_ids,
+        box_stat = box_stat,
+        box_plots = box_plots,
+        scatter_stat = scatter_stat,
+        scatter_plots = scatter_plots
+        )
+
+
+
 ### Batch Views
 
 

tests/server/test_utils.py

@@ -0,0 +1,27 @@
+
+from NIPTool.server import create_app
+from NIPTool.server.utils import get_last_batches
+from NIPTool.adapter.plugin import NiptAdapter
+from datetime import datetime
+
+
+app = create_app(test= True)
+
+
+def test_get_last_batches(database):
+    app.db = database
+    app.adapter = NiptAdapter(database.client, db_name = database.name)
+
+    # GIVEN a database with four batch documents:
+    batch1 = {"_id":"201860", "SequencingDate":"2022-03-10"}
+    batch2 = {"_id":"201862", "SequencingDate":"2022-02-10"}
+    batch3 = {"_id":"201830", "SequencingDate":"2022-02-09"}
+    batch4 = {"_id":"101830", "SequencingDate":"2021-03-10"}
+
+    database.batch.insert_many([batch4, batch1, batch3, batch2])
+
+    # WHEN running get_last_batches with nr=2
+    results = get_last_batches(adapter=app.adapter, nr=2)
+
+    # THEN the results should contain the two batches with the latest SequencingDate
+    assert results == [batch1, batch2]