Modifying interface so we can upload a QIIME mapping file

Author: josenavas

qiita_db/metadata_template/__init__.py

@@ -8,7 +8,7 @@
 
 from .sample_template import SampleTemplate
 from .prep_template import PrepTemplate
-from .util import load_template_to_dataframe
+from .util import load_template_to_dataframe, looks_like_qiime_mapping_file
 from .constants import (TARGET_GENE_DATA_TYPES, SAMPLE_TEMPLATE_COLUMNS,
                         PREP_TEMPLATE_COLUMNS,
                         PREP_TEMPLATE_COLUMNS_TARGET_GENE, CONTROLLED_COLS)
@@ -17,4 +17,4 @@
 __all__ = ['SampleTemplate', 'PrepTemplate', 'load_template_to_dataframe',
            'TARGET_GENE_DATA_TYPES', 'SAMPLE_TEMPLATE_COLUMNS',
            'PREP_TEMPLATE_COLUMNS', 'PREP_TEMPLATE_COLUMNS_TARGET_GENE',
-           'CONTROLLED_COLS']
+           'CONTROLLED_COLS', 'looks_like_qiime_mapping_file']

qiita_db/metadata_template/util.py

@@ -347,7 +347,7 @@ def looks_like_qiime_mapping_file(fp):
     '#SampleID', since a sample/prep template will start with 'sample_name' or
     some other different column.
     """
-    line = None
+    first_line = None
     with open_file(fp, mode='U') as f:
         first_line = f.readline()
     if not first_line:

qiita_pet/handlers/study_handlers/description_handlers.py

@@ -23,11 +23,14 @@
 from qiita_db.ontology import Ontology
 from qiita_db.metadata_template import (PrepTemplate, SampleTemplate,
                                         load_template_to_dataframe,
-                                        SAMPLE_TEMPLATE_COLUMNS)
+                                        SAMPLE_TEMPLATE_COLUMNS,
+                                        looks_like_qiime_mapping_file)
 from qiita_db.util import convert_to_id, get_mountpoint
 from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBColumnError,
                                  QiitaDBExecutionError, QiitaDBDuplicateError,
                                  QiitaDBDuplicateHeaderError, QiitaDBError)
+from qiita_ware.metadata_pipeline import (
+    create_templates_from_qiime_mapping_file)
 from qiita_pet.handlers.base_handlers import BaseHandler
 from qiita_pet.handlers.util import check_access
 from qiita_pet.handlers.study_handlers.listing_handlers import (
@@ -161,13 +164,11 @@ def process_sample_template(self, study, user, callback):
         HTTPError
             If the sample template file does not exists
         """
-        # If we are on this function, the argument "sample_template" must
-        # defined. If not, let tornado raise its error
+        # If we are on this function, the arguments "sample_template" and
+        # "data_type" must be defined. If not, let tornado raise its error
         sample_template = self.get_argument('sample_template')
+        data_type = self.get_argument('data_type')
 
-        # Define here the message and message level in case of success
-        msg = "The sample template '%s' has been added" % sample_template
-        msg_level = "success"
         # Get the uploads folder
         _, base_fp = get_mountpoint("uploads")[0]
         # Get the path of the sample template in the uploads folder
@@ -177,25 +178,33 @@ def process_sample_template(self, study, user, callback):
             # The file does not exist, fail nicely
             raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
 
+        # Define here the message and message level in case of success
+        is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
+
         try:
             with warnings.catch_warnings(record=True) as warns:
                 # deleting previous uploads and inserting new one
                 self.remove_add_study_template(study.raw_data, study.id,
-                                               fp_rsp)
+                                               fp_rsp, data_type,
+                                               is_mapping_file)
 
-                # join all the warning messages into one. Note that this info
-                # will be ignored if an exception is raised
+                # join all the warning messages into one. Note that this
+                # info will be ignored if an exception is raised
                 if warns:
                     msg = '; '.join([str(w.message) for w in warns])
                     msg_level = 'warning'
 
         except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
                 QiitaDBDuplicateError, IOError, ValueError, KeyError,
-                CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
+                CParserError, QiitaDBDuplicateHeaderError,
+                QiitaDBError) as e:
             # Some error occurred while processing the sample template
             # Show the error to the user so they can fix the template
-            msg = html_error_message % ('parsing the sample template:',
-                                        basename(fp_rsp), str(e))
+            error_msg = ('parsing the QIIME mapping file'
+                         if is_mapping_file
+                         else 'parsing the sample template')
+            msg = html_error_message % (error_msg, basename(fp_rsp),
+                                        str(e))
             msg_level = "danger"
 
         callback((msg, msg_level, None, None, None))
@@ -564,9 +573,14 @@ def unspecified_action(self, study, user, callback):
         msg_level = 'danger'
         callback((msg, msg_level, 'study_information_tab', None, None))
 
-    def remove_add_study_template(self, raw_data, study_id, fp_rsp):
+    def remove_add_study_template(self, raw_data, study_id, fp_rsp, data_type,
+                                  is_mapping_file):
         """Replace prep templates, raw data, and sample template with a new one
         """
+        if is_mapping_file and data_type == "":
+            raise ValueError("Please, choose a data type if uploading a QIIME "
+                             "mapping file")
+
         for rd in raw_data():
             rd = RawData(rd)
             for pt in rd.prep_templates:
@@ -575,8 +589,13 @@ def remove_add_study_template(self, raw_data, study_id, fp_rsp):
         if SampleTemplate.exists(study_id):
             SampleTemplate.delete(study_id)
 
-        SampleTemplate.create(load_template_to_dataframe(fp_rsp),
-                              Study(study_id))
+        if is_mapping_file:
+            create_templates_from_qiime_mapping_file(fp_rsp, Study(study_id),
+                                                     int(data_type))
+        else:
+            SampleTemplate.create(load_template_to_dataframe(fp_rsp),
+                                  Study(study_id))
+
         remove(fp_rsp)
 
     def remove_add_prep_template(self, fp_rpt, study, data_type_id,

qiita_pet/templates/study_description.html

@@ -67,6 +67,10 @@
         .attr("type", "hidden")
         .attr("name", "sample_template")
         .attr("value", $("#sample_template").val()))
+      .append($("<input>")
+        .attr("type", "hidden")
+        .attr("name", "data_type")
+        .attr("value", $("#qiime_data_type").val()))
       .append($("<input>")
         .attr("type", "hidden")
         .attr("name", "action")

qiita_pet/templates/study_description_templates/study_information_tab.html

@@ -12,13 +12,25 @@
   <hr>
 
   {% if show_select_sample %}
-    Select your sample template <br/>
+    Select your sample template or, alternatively, a QIIME mapping file <br/>
     (only files with the "txt" and "tsv" file extensions will be displayed here):
     <select id="sample_template">
       {% for f in files %}
         <option value="{{f}}">{{f}}</option>
       {% end %}
     </select>
+
+    <br/>
+
+    {% if not sample_templates %}
+      If you are uploading a QIIME mapping file, please choose a data type:
+      <select id="qiime_data_type">
+        <option value="">Select an option...</option>
+        {% for name, value in data_types %}
+          <option value="{{value}}">{{name}}</option>
+        {% end %}
+      </select>
+    {% end %}
   {% end %}
 
   <br/>

qiita_pet/uimodules/study_information_tab.py

@@ -7,8 +7,11 @@
 # -----------------------------------------------------------------------------
 
 from functools import partial
+from operator import itemgetter
 
-from qiita_db.util import get_files_from_uploads_folders
+from future.utils import viewitems
+
+from qiita_db.util import get_files_from_uploads_folders, get_data_types
 from qiita_db.study import StudyPerson
 from qiita_db.metadata_template import SampleTemplate
 from qiita_pet.util import linkify
@@ -34,9 +37,11 @@ def render(self, study):
         number_samples_promised = study_info['number_samples_promised']
         number_samples_collected = study_info['number_samples_collected']
         metadata_complete = study_info['metadata_complete']
+        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
 
         # Retrieve the files from the uploads folder, so the user can choose
-        # the sample template of the study
+        # the sample template of the study. Filter them to only include the
+        # ones that ends with 'txt' or 'tsv'.
         files = [f for _, f in get_files_from_uploads_folders(str(study.id))
                  if f.endswith(('txt', 'tsv'))]
 
@@ -69,4 +74,5 @@ def render(self, study):
             files=files,
             study_id=study.id,
             sample_templates=sample_templates,
-            is_local_request=is_local_request)
+            is_local_request=is_local_request,
+            data_types=data_types)

qiita_ware/metadata_pipeline.py

@@ -14,7 +14,7 @@
                                         PREP_TEMPLATE_COLUMNS_TARGET_GENE,
                                         CONTROLLED_COLS,
                                         TARGET_GENE_DATA_TYPES)
-from qiita_db.util import convert_to_id
+from qiita_db.util import convert_from_id
 
 
 def create_templates_from_qiime_mapping_file(fp, study, data_type):