diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index 20df805ee..04df30608 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -563,7 +563,7 @@ def test_update_preprocessed_data_from_cmd(self): # We need to sort the list returned from the db because the ordering # on that list is based on db modification time, rather than id obs_fps = sorted(ppd.get_filepaths()) - self.assertEqual(obs_fps, exp_fps) + self.assertEqual(obs_fps, sorted(exp_fps)) # Check that the checksums have been updated sql = "SELECT checksum FROM qiita.filepath WHERE filepath_id=%s" diff --git a/qiita_db/test/test_data.py b/qiita_db/test/test_data.py index 66cbfae81..ab383bf49 100644 --- a/qiita_db/test/test_data.py +++ b/qiita_db/test/test_data.py @@ -261,6 +261,11 @@ def test_delete(self): RawData.delete(rd.id, self.pt1.id) # Clear the files so we can actually remove the RawData + study_id = rd.studies[0] + path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id)) + self._clean_up_files.extend([join(path_for_removal, + basename(f).split('_', 1)[1]) + for _, f, _ in rd.get_filepaths()]) rd.clear_filepaths() RawData.delete(rd.id, self.pt1.id) @@ -545,7 +550,7 @@ def test_get_filepaths(self): "preprocessed_fastq"), (5, join(self.db_test_ppd_dir, '1_seqs.demux'), "preprocessed_demux")] - self.assertEqual(obs, exp) + self.assertItemsEqual(obs, exp) def test_processed_data(self): """Correctly returns the processed data id""" diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 10e2b81f4..4bb1683eb 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -575,7 +575,11 @@ def test_move_upload_files_to_trash(self): # create file to move to trash fid, folder = get_mountpoint("uploads")[0] - open(join(folder, '1', test_filename), 'w').write('test') + test_fp = join(folder, '1', test_filename) + with open(test_fp, 'w') as f: + f.write('test') + + self.files_to_remove.append(test_fp) exp = [(fid, 'this_is_a_test_file.txt'), (fid, 'uploaded_file.txt')] obs = get_files_from_uploads_folders("1") diff --git a/qiita_pet/handlers/compute.py b/qiita_pet/handlers/compute.py index 1cb16c29c..5d241b8a0 100644 --- a/qiita_pet/handlers/compute.py +++ b/qiita_pet/handlers/compute.py @@ -7,11 +7,13 @@ from .util import check_access from qiita_ware.context import submit -from qiita_ware.dispatchable import add_files_to_raw_data, unlink_all_files +from qiita_ware.dispatchable import (add_files_to_raw_data, unlink_all_files, + create_raw_data) from qiita_db.study import Study from qiita_db.exceptions import QiitaDBUnknownIDError from qiita_db.util import get_mountpoint +from qiita_db.metadata_template import PrepTemplate from os.path import join, exists @@ -28,6 +30,53 @@ def get(self, job_id): self.redirect('/') +class CreateRawData(BaseHandler): + @authenticated + def post(self): + pt_id = self.get_argument('prep_template_id') + raw_data_filetype = self.get_argument('filetype') + barcodes_str = self.get_argument('barcodes') + forward_reads_str = self.get_argument('forward') + sff_str = self.get_argument('sff') + fasta_str = self.get_argument('fasta') + qual_str = self.get_argument('qual') + reverse_reads_str = self.get_argument('reverse') + + pt = PrepTemplate(pt_id) + study_id = pt.study_id + + def _split(x): + return x.split(',') if x else [] + filepaths, fps = [], [] + fps.append((_split(barcodes_str), 'raw_barcodes')) + fps.append((_split(fasta_str), 'raw_fasta')) + fps.append((_split(qual_str), 'raw_qual')) + fps.append((_split(forward_reads_str), 'raw_forward_seqs')) + fps.append((_split(reverse_reads_str), 'raw_reverse_seqs')) + fps.append((_split(sff_str), 'raw_sff')) + + # We need to retrieve the full path for all the files, as the + # arguments only contain the file name. Since we don't know in which + # mountpoint the data lives, we retrieve all of them and we loop + # through all the files checking if they exist or not. + for _, f in get_mountpoint("uploads", retrieve_all=True): + f = join(f, str(study_id)) + for fp_set, filetype in fps: + for t in fp_set: + ft = join(f, t) + if exists(ft): + filepaths.append((ft, filetype)) + + job_id = submit(self.current_user.id, create_raw_data, + raw_data_filetype, pt, filepaths) + + self.render('compute_wait.html', + job_id=job_id, title='Adding raw data', + completion_redirect=( + '/study/description/%s?top_tab=prep_template_tab' + '&sub_tab=%s' % (study_id, pt_id))) + + class AddFilesToRawData(BaseHandler): @authenticated def post(self): @@ -35,6 +84,7 @@ def post(self): # vars to add files to raw data study_id = self.get_argument('study_id') raw_data_id = self.get_argument('raw_data_id') + prep_template_id = self.get_argument('prep_template_id') barcodes_str = self.get_argument('barcodes') forward_reads_str = self.get_argument('forward') sff_str = self.get_argument('sff') @@ -75,8 +125,8 @@ def _split(x): self.render('compute_wait.html', job_id=job_id, title='Adding files to your raw data', completion_redirect=( - '/study/description/%s?top_tab=raw_data_tab&sub_tab=%s' - % (study_id, raw_data_id))) + '/study/description/%s?top_tab=prep_template_tab' + '&sub_tab=%s' % (study_id, prep_template_id))) class UnlinkAllFiles(BaseHandler): @@ -85,6 +135,7 @@ def post(self): # vars to remove all files from a raw data study_id = self.get_argument('study_id', None) raw_data_id = self.get_argument('raw_data_id', None) + prep_template_id = self.get_argument('prep_template_id', None) study_id = int(study_id) if study_id else None @@ -101,5 +152,5 @@ def post(self): self.render('compute_wait.html', job_id=job_id, title='Removing files from your raw data', completion_redirect=( - '/study/description/%s?top_tab=raw_data_tab&sub_tab=%s' - % (study_id, raw_data_id))) + '/study/description/%s?top_tab=prep_template_tab&' + 'sub_tab=%s' % (study_id, prep_template_id))) diff --git a/qiita_pet/handlers/preprocessing_handlers.py b/qiita_pet/handlers/preprocessing_handlers.py index 6c1bb801b..46d031a73 100644 --- a/qiita_pet/handlers/preprocessing_handlers.py +++ b/qiita_pet/handlers/preprocessing_handlers.py @@ -31,6 +31,5 @@ def post(self): self.render('compute_wait.html', job_id=job_id, title='Preprocessing', completion_redirect='/study/description/%d?top_tab=' - 'raw_data_tab&sub_tab=%s&prep_tab=%s' - % (study_id, raw_data.id, - prep_template_id)) + 'prep_template_tab&sub_tab=%s' + % (study_id, prep_template_id)) diff --git a/qiita_pet/handlers/study_handlers/description_handlers.py b/qiita_pet/handlers/study_handlers/description_handlers.py index 0d45797f9..ebdc85a1a 100644 --- a/qiita_pet/handlers/study_handlers/description_handlers.py +++ b/qiita_pet/handlers/study_handlers/description_handlers.py @@ -312,8 +312,8 @@ def add_to_sample_template(self, study, user, callback): callback((msg, msg_level, None, None, None)) - def create_raw_data(self, study, user, callback): - """Adds a (new) raw data to the study + def add_raw_data(self, study, user, callback): + """Adds an existing raw data to the study Parameters ---------- @@ -328,41 +328,20 @@ def create_raw_data(self, study, user, callback): msg = "Raw data successfully added" msg_level = "success" - # Get the arguments needed to create a raw data object - filetype = self.get_argument('filetype', None) - previous_raw_data = self.get_argument('previous_raw_data', None) + # Get the arguments to add the raw data + pt_id = self.get_argument('prep_template_id') + raw_data_id = self.get_argument('raw_data_id') - if filetype and previous_raw_data: - # The user selected a filetype and an existing raw data - msg = ("You can not specify both a new raw data and a previously " - "used one") - msg_level = "danger" - elif filetype: - # We are creating a new raw data object - try: - rd_id = RawData.create(filetype, [study]).id - except (TypeError, QiitaDBColumnError, QiitaDBExecutionError, - QiitaDBDuplicateError, IOError, ValueError, KeyError, - CParserError) as e: - msg = html_error_message % ( - "creating a new raw data object for study:", - str(study.id), str(e)) - msg_level = "danger" - elif previous_raw_data: - previous_raw_data = previous_raw_data.split(',') - raw_data = [RawData(rd) for rd in previous_raw_data] - study.add_raw_data(raw_data) - rd_id = raw_data[0].id - else: - # The user did not provide a filetype neither an existing raw data - # If using the interface, we should never reach this if, but - # better be safe than sorry - msg = ("You should choose a filetype for a new raw data or " - "choose a raw data previously used") - msg_level = "danger" - rd_id = None + prep_template = PrepTemplate(pt_id) + raw_data = RawData(raw_data_id) - callback((msg, msg_level, 'raw_data_tab', rd_id, None)) + try: + prep_template.raw_data = raw_data + except QiitaDBError as e: + msg = html_error_message % ("adding the raw data", + str(raw_data_id), str(e)) + + callback((msg, msg_level, 'prep_template_tab', pt_id, None)) def add_prep_template(self, study, user, callback): """Adds a prep template to the system @@ -380,10 +359,9 @@ def add_prep_template(self, study, user, callback): msg = "Your prep template was added" msg_level = "success" - # If we are on this function, the arguments "raw_data_id", - # "prep_template" and "data_type_id" must be defined. If not, + # If we are on this function, the arguments "prep_template" and + # "data_type_id" must be defined. If not, # let tornado raise its error - raw_data_id = self.get_argument('raw_data_id') prep_template = self.get_argument('prep_template') data_type_id = self.get_argument('data_type_id') @@ -398,8 +376,6 @@ def add_prep_template(self, study, user, callback): investigation_type, user_defined_investigation_type, new_investigation_type) - # Make sure that the id is an integer - raw_data_id = _to_int(raw_data_id) # Get the upload base directory _, base_path = get_mountpoint("uploads")[0] # Get the path to the prep template @@ -414,8 +390,8 @@ def add_prep_template(self, study, user, callback): warnings.simplefilter("always") # deleting previous uploads and inserting new one - pt_id = self.remove_add_prep_template(fp_rpt, raw_data_id, - study, data_type_id, + pt_id = self.remove_add_prep_template(fp_rpt, study, + data_type_id, investigation_type) # join all the warning messages into one. Note that this info @@ -433,7 +409,7 @@ def add_prep_template(self, study, user, callback): basename(fp_rpt), str(e)) msg_level = "danger" - callback((msg, msg_level, 'raw_data_tab', raw_data_id, pt_id)) + callback((msg, msg_level, 'prep_template_tab', pt_id, None)) def make_public(self, study, user, callback): """Makes the current study public @@ -547,7 +523,6 @@ def update_investigation_type(self, study, user, callback): 'edit-new-investigation-type', None) pt = PrepTemplate(prep_id) - rd_id = pt.raw_data investigation_type = self._process_investigation_type( edit_investigation_type, edit_user_defined_investigation_type, @@ -561,9 +536,9 @@ def update_investigation_type(self, study, user, callback): msg_level = "danger" if ppd_id == 0: - top_tab = "raw_data_tab" - sub_tab = rd_id - prep_tab = prep_id + top_tab = "prep_template_tab" + sub_tab = prep_id + prep_tab = None else: top_tab = "preprocessed_data_tab" sub_tab = ppd_id @@ -604,12 +579,11 @@ def remove_add_study_template(self, raw_data, study_id, fp_rsp): Study(study_id)) remove(fp_rsp) - def remove_add_prep_template(self, fp_rpt, raw_data_id, study, - data_type_id, investigation_type): + def remove_add_prep_template(self, fp_rpt, study, data_type_id, + investigation_type): """add prep templates""" pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt), - RawData(raw_data_id), study, - _to_int(data_type_id), + study, _to_int(data_type_id), investigation_type=investigation_type).id remove(fp_rpt) return pt_id @@ -730,21 +704,18 @@ def delete_raw_data(self, study, user, callback): is done """ raw_data_id = int(self.get_argument('raw_data_id')) + prep_template_id = int(self.get_argument('prep_template_id')) try: - RawData.delete(raw_data_id, study.id) - msg = ("Raw data %d has been deleted from study: " - "%s" % (raw_data_id, study.title)) + RawData.delete(raw_data_id, prep_template_id) + msg = ("Raw data %d has been deleted from prep_template %d" + % (raw_data_id, prep_template_id)) msg_level = "success" - tab = 'study_information_tab' - tab_id = None except Exception as e: - msg = "Couldn't remove %d raw data: %s" % (raw_data_id, str(e)) + msg = "Couldn't remove raw data %d: %s" % (raw_data_id, str(e)) msg_level = "danger" - tab = 'raw_data_tab' - tab_id = raw_data_id - callback((msg, msg_level, tab, tab_id, None)) + callback((msg, msg_level, "prep_template_tab", prep_template_id, None)) def delete_prep_template(self, study, user, callback): """Delete the selected prep template @@ -771,7 +742,7 @@ def delete_prep_template(self, study, user, callback): msg = ("Couldn't remove prep template: %s" % str(e)) msg_level = "danger" - callback((msg, msg_level, 'raw_data_tab', prep_id, None)) + callback((msg, msg_level, 'prep_template_tab', prep_id, None)) def delete_preprocessed_data(self, study, user, callback): """Delete the selected preprocessed data @@ -850,7 +821,7 @@ def post(self, study_id): process_sample_template=self.process_sample_template, update_sample_template=self.update_sample_template, extend_sample_template=self.add_to_sample_template, - create_raw_data=self.create_raw_data, + add_raw_data=self.add_raw_data, add_prep_template=self.add_prep_template, make_public=self.make_public, approve_study=self.approve_study, diff --git a/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py b/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py index 249f617b7..7d05dd056 100644 --- a/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py +++ b/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py @@ -62,8 +62,8 @@ def get(self, arguments): # The prep template has been provided template = self._get_template(PrepTemplate, prep_template) back_button_path = ( - "/study/description/%s?top_tab=raw_data_tab&sub_tab=%s" - "&prep_tab=%s" % (study_id, template.raw_data, template.id)) + "/study/description/%s?top_tab=prep_template_tab&sub_tab=%s" + % (study_id, template.id)) elif sample_template: # The sample template has been provided template = self._get_template(SampleTemplate, sample_template) diff --git a/qiita_pet/templates/study_description.html b/qiita_pet/templates/study_description.html index d8f8f0de1..a9f564b45 100644 --- a/qiita_pet/templates/study_description.html +++ b/qiita_pet/templates/study_description.html @@ -111,71 +111,158 @@ form.submit(); } -function create_raw_data() { - if ($("#previous_raw_data").val() === null && $("#filetype").val() === "") { - bootstrapAlert("You need to select either a new file type or a raw data used in a previous study."); - } else if ((typeof $("#previous_raw_data").val() != 'undefined' && $("#previous_raw_data").val() !== null) && $("#filetype").val() !== "") { - bootstrapAlert("You can only select a new file type or one used in a previous study but not both."); - } else { - if ($("#filetype").val() !== "") { - var form = $("