From aca72c3c5489a9370654948c87347c02e39a897f Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 11 May 2017 10:00:25 -0600 Subject: [PATCH 1/4] init commit for redbiom-improving-listing --- qiita_pet/handlers/qiita_redbiom.py | 58 +++++++++------ qiita_pet/templates/redbiom.html | 101 +++++++++++++++++---------- qiita_pet/test/test_qiita_redbiom.py | 39 ++++++----- 3 files changed, 122 insertions(+), 76 deletions(-) diff --git a/qiita_pet/handlers/qiita_redbiom.py b/qiita_pet/handlers/qiita_redbiom.py index 74e18e1cc..e635d721a 100644 --- a/qiita_pet/handlers/qiita_redbiom.py +++ b/qiita_pet/handlers/qiita_redbiom.py @@ -83,42 +83,54 @@ def _redbiom_search(self, query, search_on, callback): (prep_template_id) JOIN qiita.study USING (study_id) WHERE sample_id IN %s - GROUP BY study_title, study_id, artifact_id) - SELECT study_title, study_id, samples, - name, command_id, - (main_query.children).artifact_id AS artifact_id - FROM main_query - JOIN qiita.artifact a ON - (main_query.children).artifact_id = a.artifact_id - JOIN qiita.artifact_type at ON ( - at.artifact_type_id = a.artifact_type_id - AND artifact_type = 'BIOM') - ORDER BY artifact_id + GROUP BY study_title, study_id, artifact_id), + artifact_query AS ( + SELECT study_title, study_id, samples, + name, command_id, + (main_query.children).artifact_id AS + artifact_id + FROM main_query + JOIN qiita.artifact a ON + (main_query.children).artifact_id = + a.artifact_id + JOIN qiita.artifact_type at ON ( + at.artifact_type_id = a.artifact_type_id + AND artifact_type = 'BIOM') + ORDER BY artifact_id) + SELECT artifact_query.*, a.command_id AS parent_cid + FROM artifact_query + LEFT JOIN qiita.parent_artifact pa ON ( + artifact_query.artifact_id = pa.artifact_id) + LEFT JOIN qiita.artifact a ON ( + pa.parent_id = a.artifact_id) """ with qdbsc.TRN: qdbsc.TRN.add(sql, [tuple(features)]) results = [] commands = {} + commands_parent = {} for row in qdbsc.TRN.execute_fetchindex(): - title, sid, samples, name, cid, aid = row + title, sid, samples, name, cid, aid, pid = row nr = {'study_title': title, 'study_id': sid, 'artifact_id': aid, 'aname': name, 'samples': samples} if cid is not None: if cid not in commands: c = qdb.software.Command(cid) - commands[cid] = { - 'sfwn': c.software.name, - 'sfv': c.software.version, - 'cmdn': c.name - } - nr['command'] = commands[cid]['cmdn'] - nr['software'] = commands[cid]['sfwn'] - nr['version'] = commands[cid]['sfv'] + commands[cid] = '%s - %s v%s' % ( + c.name, c.software.name, + c.software.version) + if pid is not None: + if pid not in commands_parent: + c = qdb.software.Command(pid) + commands_parent[pid] = c.name + nr['command'] = '%s @ %s' % ( + commands[cid], + commands_parent[pid]) + else: + nr['command'] = commands[cid] else: - nr['command'] = None - nr['software'] = None - nr['version'] = None + nr['command'] = '' results.append(nr) else: sql = """ diff --git a/qiita_pet/templates/redbiom.html b/qiita_pet/templates/redbiom.html index e83405e45..ec7f03575 100644 --- a/qiita_pet/templates/redbiom.html +++ b/qiita_pet/templates/redbiom.html @@ -38,8 +38,26 @@ { "data": null, "width": "20%" }, { "data": null, "width": "10%" } ], + "order": [[ 3, 'asc' ]], + // "displayLength": 25, + "drawCallback": function ( settings ) { + var api = this.api(); + var rows = api.rows( {page:'current'} ).nodes(); + var last=null; + + api.column(3, {page:'current'} ).data().each( function ( data, i ) { + group = data.command; + if ( last !== group ) { + $(rows).eq( i ).before( + ''+group+'' + ); + last = group; + } + }); + }, columnDefs: [ - {type:'natural', targets:[1,2,3,4]}, + { "visible": false, "targets": 3 }, + { type:'natural', targets: [1,2,4] }, // render Add to Analysis button {"render": function ( data, type, row, meta ) { {% if current_user is not None %} @@ -86,46 +104,59 @@ }, }); - $("#submitForm").submit(function(event){ - event.preventDefault(); + $("#submitForm").submit(function(event){ + event.preventDefault(); - show_loading("redbiom-info"); + show_loading("redbiom-info"); - var search = $("#search").val(); - var search_on = $("#search_on").val(); + var search = $("#search").val(); + var search_on = $("#search_on").val(); - $.post("{% raw qiita_config.portal_dir %}/redbiom/", {'search': search, 'search_on': search_on}) - .done(function ( data ){ - var redbiom_table = $('#redbiom-table').DataTable(); - var redbiom_info = $('#redbiom-info'); - redbiom_table.clear().draw(); - if(data.status == "error") { - bootstrapAlert(data.message.replace("\n", "
"), "danger"); - } else { - if(data.message != ''){ - redbiom_info.html( - `
`) - } else if(data.data){ - $('#redbiom-table tr:eq(0) th:eq(1)').text("Study Title"); - if (search_on != 'categories') { - {% if current_user is not None %} - $('#redbiom-table tr:eq(0) th:eq(0)').text("Add to Analysis"); - {% end %} - $('#redbiom-table tr:eq(0) th:eq(2)').text("Artifact Name"); - $('#redbiom-table tr:eq(0) th:eq(3)').text("Artifact Processing"); - $('#redbiom-table tr:eq(0) th:eq(4)').text("# of Samples Found"); - } else { - $('#redbiom-table tr:eq(0) th:eq(3)').text("Metadata Categories"); - $('#redbiom-table tr:eq(0) th:eq(4)').text("# of Categories Found"); - } - redbiom_table.rows.add(data.data).draw(); - redbiom_info.html(''); + $.post("{% raw qiita_config.portal_dir %}/redbiom/", {'search': search, 'search_on': search_on}) + .done(function ( data ){ + var redbiom_table = $('#redbiom-table').DataTable(); + var redbiom_info = $('#redbiom-info'); + redbiom_table.clear().draw(); + if(data.status == "error") { + bootstrapAlert(data.message.replace("\n", "
"), "danger"); + } else { + if(data.message != ''){ + redbiom_info.html( + `
`) + } else if(data.data){ + $('#redbiom-table tr:eq(0) th:eq(1)').text("Study Title"); + if (search_on != 'categories') { + {% if current_user is not None %} + $('#redbiom-table tr:eq(0) th:eq(0)').text("Add to Analysis"); + {% end %} + $('#redbiom-table tr:eq(0) th:eq(2)').text("Artifact Name"); + $('#redbiom-table tr:eq(0) th:eq(3)').text("Artifact Processing"); + $('#redbiom-table tr:eq(0) th:eq(4)').text("# of Samples Found"); + } else { + $('#redbiom-table tr:eq(0) th:eq(3)').text("Metadata Categories"); + $('#redbiom-table tr:eq(0) th:eq(4)').text("# of Categories Found"); } + redbiom_table.rows.add(data.data).draw(); + redbiom_info.html(''); } - }); + } + }); + }); + + // Order by the grouping + $('#redbiom-table tbody').on( 'click', 'tr.group', function () { + var table = $('#redbiom-table').DataTable(); + var currentOrder = table.order()[0]; + if (currentOrder[0] === 3 && currentOrder[1] === 'asc') { + table.order([ 3, 'desc' ]).draw(); + } + else { + table.order([ 3, 'asc' ]).draw(); + } }); + }); diff --git a/qiita_pet/test/test_qiita_redbiom.py b/qiita_pet/test/test_qiita_redbiom.py index 9bc7be2a3..fd052bc17 100644 --- a/qiita_pet/test/test_qiita_redbiom.py +++ b/qiita_pet/test/test_qiita_redbiom.py @@ -123,41 +123,44 @@ def test_post_errors(self): OBSERVATION = [ - {'artifact_id': 4, 'study_id': 1, 'version': '1.9.1', - 'command': 'Pick closed-reference OTUs', 'samples': [ + {'artifact_id': 4, 'study_id': 1, + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', 'samples': [ '1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198', '1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190', '1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', - 'aname': 'BIOM', 'software': 'QIIME'}, - {'artifact_id': 5, 'study_id': 1, 'version': '1.9.1', - 'command': 'Pick closed-reference OTUs', 'samples': [ + 'aname': 'BIOM'}, + {'artifact_id': 5, 'study_id': 1, + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', 'samples': [ '1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198', '1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190', '1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', - 'aname': 'BIOM', 'software': 'QIIME'}, - {'artifact_id': 6, 'study_id': 1, 'version': u'1.9.1', - 'command': 'Pick closed-reference OTUs', 'samples': [ + 'aname': 'BIOM'}, + {'artifact_id': 6, 'study_id': 1, + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', 'samples': [ '1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198', '1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190', '1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', - 'aname': 'BIOM', 'software': 'QIIME'}] + 'aname': 'BIOM'}] SEQUENCE = [ - {'artifact_id': 4, 'study_id': 1, 'version': '1.9.1', - 'command': 'Pick closed-reference OTUs', 'samples': ['1.SKM3.640197'], + {'artifact_id': 4, 'study_id': 1, + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', + 'samples': ['1.SKM3.640197'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', - 'aname': 'BIOM', 'software': 'QIIME'}, - {'artifact_id': 5, 'study_id': 1, 'version': '1.9.1', - 'command': 'Pick closed-reference OTUs', 'samples': ['1.SKM3.640197'], + 'aname': 'BIOM'}, + {'artifact_id': 5, 'study_id': 1, + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', + 'samples': ['1.SKM3.640197'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', - 'aname': 'BIOM', 'software': 'QIIME'}, - {'artifact_id': 6, 'study_id': 1, 'version': u'1.9.1', - 'command': 'Pick closed-reference OTUs', 'samples': ['1.SKM3.640197'], + 'aname': 'BIOM'}, + {'artifact_id': 6, 'study_id': 1, + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', + 'samples': ['1.SKM3.640197'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', - 'aname': 'BIOM', 'software': 'QIIME'}] + 'aname': 'BIOM'}] CATEGORIES = [ {'artifact_id': None, 'study_id': 1, 'version': None, From 5532e103bb4bac1901e830d1487b6aa2984abd94 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 12 May 2017 11:49:35 -0600 Subject: [PATCH 2/4] improving listing and grouping of artifacts --- qiita_pet/handlers/qiita_redbiom.py | 66 ++++++++++++++++++++-------- qiita_pet/templates/redbiom.html | 55 ++++++++++++----------- qiita_pet/test/test_qiita_redbiom.py | 15 ++++--- 3 files changed, 83 insertions(+), 53 deletions(-) diff --git a/qiita_pet/handlers/qiita_redbiom.py b/qiita_pet/handlers/qiita_redbiom.py index e635d721a..e3f572e82 100644 --- a/qiita_pet/handlers/qiita_redbiom.py +++ b/qiita_pet/handlers/qiita_redbiom.py @@ -6,6 +6,7 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- +from future.utils import viewitems from requests import ConnectionError import redbiom.summarize import redbiom.search @@ -96,21 +97,45 @@ def _redbiom_search(self, query, search_on, callback): JOIN qiita.artifact_type at ON ( at.artifact_type_id = a.artifact_type_id AND artifact_type = 'BIOM') - ORDER BY artifact_id) - SELECT artifact_query.*, a.command_id AS parent_cid - FROM artifact_query - LEFT JOIN qiita.parent_artifact pa ON ( - artifact_query.artifact_id = pa.artifact_id) - LEFT JOIN qiita.artifact a ON ( - pa.parent_id = a.artifact_id) + ORDER BY artifact_id), + parent_query AS ( + SELECT artifact_query.*, + array_agg(parent_params) as parent_parameters + FROM artifact_query + LEFT JOIN qiita.parent_artifact pa ON ( + artifact_query.artifact_id = pa.artifact_id) + LEFT JOIN qiita.artifact a ON ( + pa.parent_id = a.artifact_id), + json_each_text(command_parameters) + parent_params + GROUP BY artifact_query.study_title, + artifact_query.study_id, + artifact_query.samples, artifact_query.name, + artifact_query.command_id, + artifact_query.artifact_id) + SELECT * FROM parent_query """ + + sql_params = """ + SELECT parameter_set_name, array_agg(ps) AS param_set + FROM qiita.default_parameter_set, + json_each_text(parameter_set) ps + GROUP BY parameter_set_name""" + with qdbsc.TRN: - qdbsc.TRN.add(sql, [tuple(features)]) results = [] commands = {} - commands_parent = {} + # obtaining all existing parameters, note that + # they are not that many (~40) and we don't expect + # to have a huge growth in the near future + qdbsc.TRN.add(sql_params) + params = {pname: eval(params) for pname, params + in qdbsc.TRN.execute_fetchindex()} + + # now let's get the actual artifacts + qdbsc.TRN.add(sql, [tuple(features)]) for row in qdbsc.TRN.execute_fetchindex(): - title, sid, samples, name, cid, aid, pid = row + title, sid, samples, name, cid, aid, pp = row nr = {'study_title': title, 'study_id': sid, 'artifact_id': aid, 'aname': name, 'samples': samples} @@ -120,15 +145,18 @@ def _redbiom_search(self, query, search_on, callback): commands[cid] = '%s - %s v%s' % ( c.name, c.software.name, c.software.version) - if pid is not None: - if pid not in commands_parent: - c = qdb.software.Command(pid) - commands_parent[pid] = c.name - nr['command'] = '%s @ %s' % ( - commands[cid], - commands_parent[pid]) - else: - nr['command'] = commands[cid] + + # [-1] taking the last cause it's sorted by + # the number of overlapping parameters + # [0] then taking the first element that is + # the name of the parameter set + ppc = sorted( + [[k, len(eval(pp) & v)] + for k, v in viewitems(params)], + key=lambda x: x[1])[-1][0] + + nr['command'] = '%s @ %s' % ( + commands[cid], ppc) else: nr['command'] = '' results.append(nr) diff --git a/qiita_pet/templates/redbiom.html b/qiita_pet/templates/redbiom.html index ec7f03575..23ab1844b 100644 --- a/qiita_pet/templates/redbiom.html +++ b/qiita_pet/templates/redbiom.html @@ -60,11 +60,12 @@ { type:'natural', targets: [1,2,4] }, // render Add to Analysis button {"render": function ( data, type, row, meta ) { + var text = ''; {% if current_user is not None %} - var text = ''; - {% else %} - var text = ''; + if ($("#search_on").val() != 'categories'){ + var text = ''; + } {% end %} return text; }, targets: [0]}, @@ -78,21 +79,15 @@ {% end %} return text; }, targets: [1]}, - // render Artifact Processing + // render Artifact Name or Metadata Categories {"render": function ( data, type, row, meta ) { - var text = ''; - if (!(!row.command)) { - text += row.command; - } - if (!(!row.software)) { - text += ' ' + row.software; - } - if (!(!row.version)) { - text += ' v' + row.version; - } + var text = row.aname; + if ($("#search_on").val() == 'categories'){ + text = row.samples.join(', '); + } return text; - }, targets: [3]}, - // render Artifact Processing + }, targets: [2]}, + // render # of samples/categories {"render": function ( data, type, row, meta ) { return row.samples.length; }, targets: [4]} @@ -126,18 +121,22 @@ Warning! ` + data.message + `
`) } else if(data.data){ - $('#redbiom-table tr:eq(0) th:eq(1)').text("Study Title"); - if (search_on != 'categories') { - {% if current_user is not None %} - $('#redbiom-table tr:eq(0) th:eq(0)').text("Add to Analysis"); - {% end %} - $('#redbiom-table tr:eq(0) th:eq(2)').text("Artifact Name"); - $('#redbiom-table tr:eq(0) th:eq(3)').text("Artifact Processing"); - $('#redbiom-table tr:eq(0) th:eq(4)').text("# of Samples Found"); - } else { - $('#redbiom-table tr:eq(0) th:eq(3)').text("Metadata Categories"); - $('#redbiom-table tr:eq(0) th:eq(4)').text("# of Categories Found"); + {% if current_user is not None %} + var header_0 = "Add to Analysis"; + {% else %} + var header_0 = ""; + {% end %} + var header_2 = "Artifact Name"; + var header_3 = "# of Samples Found"; + if (search_on == 'categories') { + header_2 = "Metadata Categories"; + header_3 = "# of Categories Found"; } + $('#redbiom-table tr:eq(0) th:eq(0)').text(header_0); + $('#redbiom-table tr:eq(0) th:eq(1)').text("Study Title"); + $('#redbiom-table tr:eq(0) th:eq(2)').text(header_2); + $('#redbiom-table tr:eq(0) th:eq(3)').text(header_3); + redbiom_table.rows.add(data.data).draw(); redbiom_info.html(''); } diff --git a/qiita_pet/test/test_qiita_redbiom.py b/qiita_pet/test/test_qiita_redbiom.py index fd052bc17..735748325 100644 --- a/qiita_pet/test/test_qiita_redbiom.py +++ b/qiita_pet/test/test_qiita_redbiom.py @@ -124,21 +124,24 @@ def test_post_errors(self): OBSERVATION = [ {'artifact_id': 4, 'study_id': 1, - 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', 'samples': [ + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Defaults', + 'samples': [ '1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198', '1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190', '1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', 'aname': 'BIOM'}, {'artifact_id': 5, 'study_id': 1, - 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', 'samples': [ + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Defaults', + 'samples': [ '1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198', '1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190', '1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', 'aname': 'BIOM'}, {'artifact_id': 6, 'study_id': 1, - 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', 'samples': [ + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Defaults', + 'samples': [ '1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198', '1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190', '1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182'], @@ -147,17 +150,17 @@ def test_post_errors(self): SEQUENCE = [ {'artifact_id': 4, 'study_id': 1, - 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Defaults', 'samples': ['1.SKM3.640197'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', 'aname': 'BIOM'}, {'artifact_id': 5, 'study_id': 1, - 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Defaults', 'samples': ['1.SKM3.640197'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', 'aname': 'BIOM'}, {'artifact_id': 6, 'study_id': 1, - 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Split libraries FASTQ', + 'command': 'Pick closed-reference OTUs - QIIME v1.9.1 @ Defaults', 'samples': ['1.SKM3.640197'], 'study_title': 'Identification of the Microbiomes for Cannabis Soils', 'aname': 'BIOM'}] From 58edff22ee465a88d6201ab46ec3377cb3f88802 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 12 May 2017 12:08:49 -0600 Subject: [PATCH 3/4] show_loading needs to be in sitebase.html cause o the import --- qiita_pet/static/js/qiita.js | 14 -------------- qiita_pet/templates/sitebase.html | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/qiita_pet/static/js/qiita.js b/qiita_pet/static/js/qiita.js index a2a20c6e7..04c9b199a 100644 --- a/qiita_pet/static/js/qiita.js +++ b/qiita_pet/static/js/qiita.js @@ -301,17 +301,3 @@ function display_number_of_samples_added(num_samples) { bootstrapAlert(num_samples + ' samples selected.', "success", 10000); $('#dflt-sel-info').css('color', 'rgb(0, 160, 0)'); } - -/** - * - * Function to show the loading gif in a given div - * - * @param div_name string with the name of the div to populate with the loading gif - * - * This function replaces the content of the given div with the - * gif to show that the section of page is loading - * - */ -function show_loading(div_name) { - $("#" + div_name).html(""); -} diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index b5db60673..9921c8faf 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -54,6 +54,20 @@