Merge pull request #1207 from josenavas/1084-qiita-pet

1084 qiita pet

qiita_db/test/test_commands.py

@@ -563,7 +563,7 @@ def test_update_preprocessed_data_from_cmd(self):
         # We need to sort the list returned from the db because the ordering
         # on that list is based on db modification time, rather than id
         obs_fps = sorted(ppd.get_filepaths())
-        self.assertEqual(obs_fps, exp_fps)
+        self.assertEqual(obs_fps, sorted(exp_fps))
 
         # Check that the checksums have been updated
         sql = "SELECT checksum FROM qiita.filepath WHERE filepath_id=%s"

qiita_db/test/test_data.py

@@ -261,6 +261,11 @@ def test_delete(self):
             RawData.delete(rd.id, self.pt1.id)
 
         # Clear the files so we can actually remove the RawData
+        study_id = rd.studies[0]
+        path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id))
+        self._clean_up_files.extend([join(path_for_removal,
+                                     basename(f).split('_', 1)[1])
+                                    for _, f, _ in rd.get_filepaths()])
         rd.clear_filepaths()
 
         RawData.delete(rd.id, self.pt1.id)
@@ -545,7 +550,7 @@ def test_get_filepaths(self):
                 "preprocessed_fastq"),
                (5, join(self.db_test_ppd_dir, '1_seqs.demux'),
                 "preprocessed_demux")]
-        self.assertEqual(obs, exp)
+        self.assertItemsEqual(obs, exp)
 
     def test_processed_data(self):
         """Correctly returns the processed data id"""

qiita_db/test/test_util.py

@@ -575,7 +575,11 @@ def test_move_upload_files_to_trash(self):
 
         # create file to move to trash
         fid, folder = get_mountpoint("uploads")[0]
-        open(join(folder, '1', test_filename), 'w').write('test')
+        test_fp = join(folder, '1', test_filename)
+        with open(test_fp, 'w') as f:
+            f.write('test')
+
+        self.files_to_remove.append(test_fp)
 
         exp = [(fid, 'this_is_a_test_file.txt'), (fid, 'uploaded_file.txt')]
         obs = get_files_from_uploads_folders("1")

qiita_pet/handlers/compute.py

@@ -7,11 +7,13 @@
 from .util import check_access
 
 from qiita_ware.context import submit
-from qiita_ware.dispatchable import add_files_to_raw_data, unlink_all_files
+from qiita_ware.dispatchable import (add_files_to_raw_data, unlink_all_files,
+                                     create_raw_data)
 
 from qiita_db.study import Study
 from qiita_db.exceptions import QiitaDBUnknownIDError
 from qiita_db.util import get_mountpoint
+from qiita_db.metadata_template import PrepTemplate
 
 from os.path import join, exists
 
@@ -28,13 +30,61 @@ def get(self, job_id):
         self.redirect('/')
 
 
+class CreateRawData(BaseHandler):
+    @authenticated
+    def post(self):
+        pt_id = self.get_argument('prep_template_id')
+        raw_data_filetype = self.get_argument('filetype')
+        barcodes_str = self.get_argument('barcodes')
+        forward_reads_str = self.get_argument('forward')
+        sff_str = self.get_argument('sff')
+        fasta_str = self.get_argument('fasta')
+        qual_str = self.get_argument('qual')
+        reverse_reads_str = self.get_argument('reverse')
+
+        pt = PrepTemplate(pt_id)
+        study_id = pt.study_id
+
+        def _split(x):
+            return x.split(',') if x else []
+        filepaths, fps = [], []
+        fps.append((_split(barcodes_str), 'raw_barcodes'))
+        fps.append((_split(fasta_str), 'raw_fasta'))
+        fps.append((_split(qual_str), 'raw_qual'))
+        fps.append((_split(forward_reads_str), 'raw_forward_seqs'))
+        fps.append((_split(reverse_reads_str), 'raw_reverse_seqs'))
+        fps.append((_split(sff_str), 'raw_sff'))
+
+        # We need to retrieve the full path for all the files, as the
+        # arguments only contain the file name. Since we don't know in which
+        # mountpoint the data lives, we retrieve all of them and we loop
+        # through all the files checking if they exist or not.
+        for _, f in get_mountpoint("uploads", retrieve_all=True):
+            f = join(f, str(study_id))
+            for fp_set, filetype in fps:
+                for t in fp_set:
+                    ft = join(f, t)
+                    if exists(ft):
+                        filepaths.append((ft, filetype))
+
+        job_id = submit(self.current_user.id, create_raw_data,
+                        raw_data_filetype, pt, filepaths)
+
+        self.render('compute_wait.html',
+                    job_id=job_id, title='Adding raw data',
+                    completion_redirect=(
+                        '/study/description/%s?top_tab=prep_template_tab'
+                        '&sub_tab=%s' % (study_id, pt_id)))
+
+
 class AddFilesToRawData(BaseHandler):
     @authenticated
     def post(self):
 
         # vars to add files to raw data
         study_id = self.get_argument('study_id')
         raw_data_id = self.get_argument('raw_data_id')
+        prep_template_id = self.get_argument('prep_template_id')
         barcodes_str = self.get_argument('barcodes')
         forward_reads_str = self.get_argument('forward')
         sff_str = self.get_argument('sff')
@@ -75,8 +125,8 @@ def _split(x):
         self.render('compute_wait.html',
                     job_id=job_id, title='Adding files to your raw data',
                     completion_redirect=(
-                        '/study/description/%s?top_tab=raw_data_tab&sub_tab=%s'
-                        % (study_id, raw_data_id)))
+                        '/study/description/%s?top_tab=prep_template_tab'
+                        '&sub_tab=%s' % (study_id, prep_template_id)))
 
 
 class UnlinkAllFiles(BaseHandler):
@@ -85,6 +135,7 @@ def post(self):
         # vars to remove all files from a raw data
         study_id = self.get_argument('study_id', None)
         raw_data_id = self.get_argument('raw_data_id', None)
+        prep_template_id = self.get_argument('prep_template_id', None)
 
         study_id = int(study_id) if study_id else None
 
@@ -101,5 +152,5 @@ def post(self):
         self.render('compute_wait.html', job_id=job_id,
                     title='Removing files from your raw data',
                     completion_redirect=(
-                        '/study/description/%s?top_tab=raw_data_tab&sub_tab=%s'
-                        % (study_id, raw_data_id)))
+                        '/study/description/%s?top_tab=prep_template_tab&'
+                        'sub_tab=%s' % (study_id, prep_template_id)))

qiita_pet/handlers/preprocessing_handlers.py

@@ -31,6 +31,5 @@ def post(self):
         self.render('compute_wait.html',
                     job_id=job_id, title='Preprocessing',
                     completion_redirect='/study/description/%d?top_tab='
-                                        'raw_data_tab&sub_tab=%s&prep_tab=%s'
-                                        % (study_id, raw_data.id,
-                                           prep_template_id))
+                                        'prep_template_tab&sub_tab=%s'
+                                        % (study_id, prep_template_id))

qiita_pet/handlers/study_handlers/description_handlers.py

@@ -312,8 +312,8 @@ def add_to_sample_template(self, study, user, callback):
 
         callback((msg, msg_level, None, None, None))
 
-    def create_raw_data(self, study, user, callback):
-        """Adds a (new) raw data to the study
+    def add_raw_data(self, study, user, callback):
+        """Adds an existing raw data to the study
 
         Parameters
         ----------
@@ -328,41 +328,20 @@ def create_raw_data(self, study, user, callback):
         msg = "Raw data successfully added"
         msg_level = "success"
 
-        # Get the arguments needed to create a raw data object
-        filetype = self.get_argument('filetype', None)
-        previous_raw_data = self.get_argument('previous_raw_data', None)
+        # Get the arguments to add the raw data
+        pt_id = self.get_argument('prep_template_id')
+        raw_data_id = self.get_argument('raw_data_id')
 
-        if filetype and previous_raw_data:
-            # The user selected a filetype and an existing raw data
-            msg = ("You can not specify both a new raw data and a previously "
-                   "used one")
-            msg_level = "danger"
-        elif filetype:
-            # We are creating a new raw data object
-            try:
-                rd_id = RawData.create(filetype, [study]).id
-            except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
-                    QiitaDBDuplicateError, IOError, ValueError, KeyError,
-                    CParserError) as e:
-                msg = html_error_message % (
-                    "creating a new raw data object for study:",
-                    str(study.id), str(e))
-                msg_level = "danger"
-        elif previous_raw_data:
-            previous_raw_data = previous_raw_data.split(',')
-            raw_data = [RawData(rd) for rd in previous_raw_data]
-            study.add_raw_data(raw_data)
-            rd_id = raw_data[0].id
-        else:
-            # The user did not provide a filetype neither an existing raw data
-            # If using the interface, we should never reach this if, but
-            # better be safe than sorry
-            msg = ("You should choose a filetype for a new raw data or "
-                   "choose a raw data previously used")
-            msg_level = "danger"
-            rd_id = None
+        prep_template = PrepTemplate(pt_id)
+        raw_data = RawData(raw_data_id)
 
-        callback((msg, msg_level, 'raw_data_tab', rd_id, None))
+        try:
+            prep_template.raw_data = raw_data
+        except QiitaDBError as e:
+            msg = html_error_message % ("adding the raw data",
+                                        str(raw_data_id), str(e))
+
+        callback((msg, msg_level, 'prep_template_tab', pt_id, None))
 
     def add_prep_template(self, study, user, callback):
         """Adds a prep template to the system
@@ -380,10 +359,9 @@ def add_prep_template(self, study, user, callback):
         msg = "Your prep template was added"
         msg_level = "success"
 
-        # If we are on this function, the arguments "raw_data_id",
-        # "prep_template" and "data_type_id" must be defined. If not,
+        # If we are on this function, the arguments "prep_template" and
+        # "data_type_id" must be defined. If not,
         # let tornado raise its error
-        raw_data_id = self.get_argument('raw_data_id')
         prep_template = self.get_argument('prep_template')
         data_type_id = self.get_argument('data_type_id')
 
@@ -398,8 +376,6 @@ def add_prep_template(self, study, user, callback):
             investigation_type, user_defined_investigation_type,
             new_investigation_type)
 
-        # Make sure that the id is an integer
-        raw_data_id = _to_int(raw_data_id)
         # Get the upload base directory
         _, base_path = get_mountpoint("uploads")[0]
         # Get the path to the prep template
@@ -414,8 +390,8 @@ def add_prep_template(self, study, user, callback):
                 warnings.simplefilter("always")
 
                 # deleting previous uploads and inserting new one
-                pt_id = self.remove_add_prep_template(fp_rpt, raw_data_id,
-                                                      study, data_type_id,
+                pt_id = self.remove_add_prep_template(fp_rpt, study,
+                                                      data_type_id,
                                                       investigation_type)
 
                 # join all the warning messages into one. Note that this info
@@ -433,7 +409,7 @@ def add_prep_template(self, study, user, callback):
                                         basename(fp_rpt), str(e))
             msg_level = "danger"
 
-        callback((msg, msg_level, 'raw_data_tab', raw_data_id, pt_id))
+        callback((msg, msg_level, 'prep_template_tab', pt_id, None))
 
     def make_public(self, study, user, callback):
         """Makes the current study public
@@ -547,7 +523,6 @@ def update_investigation_type(self, study, user, callback):
             'edit-new-investigation-type', None)
 
         pt = PrepTemplate(prep_id)
-        rd_id = pt.raw_data
 
         investigation_type = self._process_investigation_type(
             edit_investigation_type, edit_user_defined_investigation_type,
@@ -561,9 +536,9 @@ def update_investigation_type(self, study, user, callback):
             msg_level = "danger"
 
         if ppd_id == 0:
-            top_tab = "raw_data_tab"
-            sub_tab = rd_id
-            prep_tab = prep_id
+            top_tab = "prep_template_tab"
+            sub_tab = prep_id
+            prep_tab = None
         else:
             top_tab = "preprocessed_data_tab"
             sub_tab = ppd_id
@@ -604,12 +579,11 @@ def remove_add_study_template(self, raw_data, study_id, fp_rsp):
                               Study(study_id))
         remove(fp_rsp)
 
-    def remove_add_prep_template(self, fp_rpt, raw_data_id, study,
-                                 data_type_id, investigation_type):
+    def remove_add_prep_template(self, fp_rpt, study, data_type_id,
+                                 investigation_type):
         """add prep templates"""
         pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt),
-                                    RawData(raw_data_id), study,
-                                    _to_int(data_type_id),
+                                    study, _to_int(data_type_id),
                                     investigation_type=investigation_type).id
         remove(fp_rpt)
         return pt_id
@@ -730,21 +704,18 @@ def delete_raw_data(self, study, user, callback):
             is done
         """
         raw_data_id = int(self.get_argument('raw_data_id'))
+        prep_template_id = int(self.get_argument('prep_template_id'))
 
         try:
-            RawData.delete(raw_data_id, study.id)
-            msg = ("Raw data %d has been deleted from study: "
-                   "<b><i>%s</i></b>" % (raw_data_id, study.title))
+            RawData.delete(raw_data_id, prep_template_id)
+            msg = ("Raw data %d has been deleted from prep_template %d"
+                   % (raw_data_id, prep_template_id))
             msg_level = "success"
-            tab = 'study_information_tab'
-            tab_id = None
         except Exception as e:
-            msg = "Couldn't remove %d raw data: %s" % (raw_data_id, str(e))
+            msg = "Couldn't remove raw data %d: %s" % (raw_data_id, str(e))
             msg_level = "danger"
-            tab = 'raw_data_tab'
-            tab_id = raw_data_id
 
-        callback((msg, msg_level, tab, tab_id, None))
+        callback((msg, msg_level, "prep_template_tab", prep_template_id, None))
 
     def delete_prep_template(self, study, user, callback):
         """Delete the selected prep template
@@ -771,7 +742,7 @@ def delete_prep_template(self, study, user, callback):
             msg = ("Couldn't remove prep template: %s" % str(e))
             msg_level = "danger"
 
-        callback((msg, msg_level, 'raw_data_tab', prep_id, None))
+        callback((msg, msg_level, 'prep_template_tab', prep_id, None))
 
     def delete_preprocessed_data(self, study, user, callback):
         """Delete the selected preprocessed data
@@ -850,7 +821,7 @@ def post(self, study_id):
             process_sample_template=self.process_sample_template,
             update_sample_template=self.update_sample_template,
             extend_sample_template=self.add_to_sample_template,
-            create_raw_data=self.create_raw_data,
+            add_raw_data=self.add_raw_data,
             add_prep_template=self.add_prep_template,
             make_public=self.make_public,
             approve_study=self.approve_study,

qiita_pet/handlers/study_handlers/metadata_summary_handlers.py

@@ -62,8 +62,8 @@ def get(self, arguments):
             # The prep template has been provided
             template = self._get_template(PrepTemplate, prep_template)
             back_button_path = (
-                "/study/description/%s?top_tab=raw_data_tab&sub_tab=%s"
-                "&prep_tab=%s" % (study_id, template.raw_data, template.id))
+                "/study/description/%s?top_tab=prep_template_tab&sub_tab=%s"
+                % (study_id, template.id))
         elif sample_template:
             # The sample template has been provided
             template = self._get_template(SampleTemplate, sample_template)