diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index 20df805ee..04df30608 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -563,7 +563,7 @@ def test_update_preprocessed_data_from_cmd(self): # We need to sort the list returned from the db because the ordering # on that list is based on db modification time, rather than id obs_fps = sorted(ppd.get_filepaths()) - self.assertEqual(obs_fps, exp_fps) + self.assertEqual(obs_fps, sorted(exp_fps)) # Check that the checksums have been updated sql = "SELECT checksum FROM qiita.filepath WHERE filepath_id=%s" diff --git a/qiita_db/test/test_data.py b/qiita_db/test/test_data.py index 66cbfae81..ab383bf49 100644 --- a/qiita_db/test/test_data.py +++ b/qiita_db/test/test_data.py @@ -261,6 +261,11 @@ def test_delete(self): RawData.delete(rd.id, self.pt1.id) # Clear the files so we can actually remove the RawData + study_id = rd.studies[0] + path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id)) + self._clean_up_files.extend([join(path_for_removal, + basename(f).split('_', 1)[1]) + for _, f, _ in rd.get_filepaths()]) rd.clear_filepaths() RawData.delete(rd.id, self.pt1.id) @@ -545,7 +550,7 @@ def test_get_filepaths(self): "preprocessed_fastq"), (5, join(self.db_test_ppd_dir, '1_seqs.demux'), "preprocessed_demux")] - self.assertEqual(obs, exp) + self.assertItemsEqual(obs, exp) def test_processed_data(self): """Correctly returns the processed data id""" diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 10e2b81f4..4bb1683eb 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -575,7 +575,11 @@ def test_move_upload_files_to_trash(self): # create file to move to trash fid, folder = get_mountpoint("uploads")[0] - open(join(folder, '1', test_filename), 'w').write('test') + test_fp = join(folder, '1', test_filename) + with open(test_fp, 'w') as f: + f.write('test') + + self.files_to_remove.append(test_fp) exp = [(fid, 'this_is_a_test_file.txt'), (fid, 'uploaded_file.txt')] obs = get_files_from_uploads_folders("1") diff --git a/qiita_pet/handlers/compute.py b/qiita_pet/handlers/compute.py index 1cb16c29c..5d241b8a0 100644 --- a/qiita_pet/handlers/compute.py +++ b/qiita_pet/handlers/compute.py @@ -7,11 +7,13 @@ from .util import check_access from qiita_ware.context import submit -from qiita_ware.dispatchable import add_files_to_raw_data, unlink_all_files +from qiita_ware.dispatchable import (add_files_to_raw_data, unlink_all_files, + create_raw_data) from qiita_db.study import Study from qiita_db.exceptions import QiitaDBUnknownIDError from qiita_db.util import get_mountpoint +from qiita_db.metadata_template import PrepTemplate from os.path import join, exists @@ -28,6 +30,53 @@ def get(self, job_id): self.redirect('/') +class CreateRawData(BaseHandler): + @authenticated + def post(self): + pt_id = self.get_argument('prep_template_id') + raw_data_filetype = self.get_argument('filetype') + barcodes_str = self.get_argument('barcodes') + forward_reads_str = self.get_argument('forward') + sff_str = self.get_argument('sff') + fasta_str = self.get_argument('fasta') + qual_str = self.get_argument('qual') + reverse_reads_str = self.get_argument('reverse') + + pt = PrepTemplate(pt_id) + study_id = pt.study_id + + def _split(x): + return x.split(',') if x else [] + filepaths, fps = [], [] + fps.append((_split(barcodes_str), 'raw_barcodes')) + fps.append((_split(fasta_str), 'raw_fasta')) + fps.append((_split(qual_str), 'raw_qual')) + fps.append((_split(forward_reads_str), 'raw_forward_seqs')) + fps.append((_split(reverse_reads_str), 'raw_reverse_seqs')) + fps.append((_split(sff_str), 'raw_sff')) + + # We need to retrieve the full path for all the files, as the + # arguments only contain the file name. Since we don't know in which + # mountpoint the data lives, we retrieve all of them and we loop + # through all the files checking if they exist or not. + for _, f in get_mountpoint("uploads", retrieve_all=True): + f = join(f, str(study_id)) + for fp_set, filetype in fps: + for t in fp_set: + ft = join(f, t) + if exists(ft): + filepaths.append((ft, filetype)) + + job_id = submit(self.current_user.id, create_raw_data, + raw_data_filetype, pt, filepaths) + + self.render('compute_wait.html', + job_id=job_id, title='Adding raw data', + completion_redirect=( + '/study/description/%s?top_tab=prep_template_tab' + '&sub_tab=%s' % (study_id, pt_id))) + + class AddFilesToRawData(BaseHandler): @authenticated def post(self): @@ -35,6 +84,7 @@ def post(self): # vars to add files to raw data study_id = self.get_argument('study_id') raw_data_id = self.get_argument('raw_data_id') + prep_template_id = self.get_argument('prep_template_id') barcodes_str = self.get_argument('barcodes') forward_reads_str = self.get_argument('forward') sff_str = self.get_argument('sff') @@ -75,8 +125,8 @@ def _split(x): self.render('compute_wait.html', job_id=job_id, title='Adding files to your raw data', completion_redirect=( - '/study/description/%s?top_tab=raw_data_tab&sub_tab=%s' - % (study_id, raw_data_id))) + '/study/description/%s?top_tab=prep_template_tab' + '&sub_tab=%s' % (study_id, prep_template_id))) class UnlinkAllFiles(BaseHandler): @@ -85,6 +135,7 @@ def post(self): # vars to remove all files from a raw data study_id = self.get_argument('study_id', None) raw_data_id = self.get_argument('raw_data_id', None) + prep_template_id = self.get_argument('prep_template_id', None) study_id = int(study_id) if study_id else None @@ -101,5 +152,5 @@ def post(self): self.render('compute_wait.html', job_id=job_id, title='Removing files from your raw data', completion_redirect=( - '/study/description/%s?top_tab=raw_data_tab&sub_tab=%s' - % (study_id, raw_data_id))) + '/study/description/%s?top_tab=prep_template_tab&' + 'sub_tab=%s' % (study_id, prep_template_id))) diff --git a/qiita_pet/handlers/preprocessing_handlers.py b/qiita_pet/handlers/preprocessing_handlers.py index 6c1bb801b..46d031a73 100644 --- a/qiita_pet/handlers/preprocessing_handlers.py +++ b/qiita_pet/handlers/preprocessing_handlers.py @@ -31,6 +31,5 @@ def post(self): self.render('compute_wait.html', job_id=job_id, title='Preprocessing', completion_redirect='/study/description/%d?top_tab=' - 'raw_data_tab&sub_tab=%s&prep_tab=%s' - % (study_id, raw_data.id, - prep_template_id)) + 'prep_template_tab&sub_tab=%s' + % (study_id, prep_template_id)) diff --git a/qiita_pet/handlers/study_handlers/description_handlers.py b/qiita_pet/handlers/study_handlers/description_handlers.py index 0d45797f9..ebdc85a1a 100644 --- a/qiita_pet/handlers/study_handlers/description_handlers.py +++ b/qiita_pet/handlers/study_handlers/description_handlers.py @@ -312,8 +312,8 @@ def add_to_sample_template(self, study, user, callback): callback((msg, msg_level, None, None, None)) - def create_raw_data(self, study, user, callback): - """Adds a (new) raw data to the study + def add_raw_data(self, study, user, callback): + """Adds an existing raw data to the study Parameters ---------- @@ -328,41 +328,20 @@ def create_raw_data(self, study, user, callback): msg = "Raw data successfully added" msg_level = "success" - # Get the arguments needed to create a raw data object - filetype = self.get_argument('filetype', None) - previous_raw_data = self.get_argument('previous_raw_data', None) + # Get the arguments to add the raw data + pt_id = self.get_argument('prep_template_id') + raw_data_id = self.get_argument('raw_data_id') - if filetype and previous_raw_data: - # The user selected a filetype and an existing raw data - msg = ("You can not specify both a new raw data and a previously " - "used one") - msg_level = "danger" - elif filetype: - # We are creating a new raw data object - try: - rd_id = RawData.create(filetype, [study]).id - except (TypeError, QiitaDBColumnError, QiitaDBExecutionError, - QiitaDBDuplicateError, IOError, ValueError, KeyError, - CParserError) as e: - msg = html_error_message % ( - "creating a new raw data object for study:", - str(study.id), str(e)) - msg_level = "danger" - elif previous_raw_data: - previous_raw_data = previous_raw_data.split(',') - raw_data = [RawData(rd) for rd in previous_raw_data] - study.add_raw_data(raw_data) - rd_id = raw_data[0].id - else: - # The user did not provide a filetype neither an existing raw data - # If using the interface, we should never reach this if, but - # better be safe than sorry - msg = ("You should choose a filetype for a new raw data or " - "choose a raw data previously used") - msg_level = "danger" - rd_id = None + prep_template = PrepTemplate(pt_id) + raw_data = RawData(raw_data_id) - callback((msg, msg_level, 'raw_data_tab', rd_id, None)) + try: + prep_template.raw_data = raw_data + except QiitaDBError as e: + msg = html_error_message % ("adding the raw data", + str(raw_data_id), str(e)) + + callback((msg, msg_level, 'prep_template_tab', pt_id, None)) def add_prep_template(self, study, user, callback): """Adds a prep template to the system @@ -380,10 +359,9 @@ def add_prep_template(self, study, user, callback): msg = "Your prep template was added" msg_level = "success" - # If we are on this function, the arguments "raw_data_id", - # "prep_template" and "data_type_id" must be defined. If not, + # If we are on this function, the arguments "prep_template" and + # "data_type_id" must be defined. If not, # let tornado raise its error - raw_data_id = self.get_argument('raw_data_id') prep_template = self.get_argument('prep_template') data_type_id = self.get_argument('data_type_id') @@ -398,8 +376,6 @@ def add_prep_template(self, study, user, callback): investigation_type, user_defined_investigation_type, new_investigation_type) - # Make sure that the id is an integer - raw_data_id = _to_int(raw_data_id) # Get the upload base directory _, base_path = get_mountpoint("uploads")[0] # Get the path to the prep template @@ -414,8 +390,8 @@ def add_prep_template(self, study, user, callback): warnings.simplefilter("always") # deleting previous uploads and inserting new one - pt_id = self.remove_add_prep_template(fp_rpt, raw_data_id, - study, data_type_id, + pt_id = self.remove_add_prep_template(fp_rpt, study, + data_type_id, investigation_type) # join all the warning messages into one. Note that this info @@ -433,7 +409,7 @@ def add_prep_template(self, study, user, callback): basename(fp_rpt), str(e)) msg_level = "danger" - callback((msg, msg_level, 'raw_data_tab', raw_data_id, pt_id)) + callback((msg, msg_level, 'prep_template_tab', pt_id, None)) def make_public(self, study, user, callback): """Makes the current study public @@ -547,7 +523,6 @@ def update_investigation_type(self, study, user, callback): 'edit-new-investigation-type', None) pt = PrepTemplate(prep_id) - rd_id = pt.raw_data investigation_type = self._process_investigation_type( edit_investigation_type, edit_user_defined_investigation_type, @@ -561,9 +536,9 @@ def update_investigation_type(self, study, user, callback): msg_level = "danger" if ppd_id == 0: - top_tab = "raw_data_tab" - sub_tab = rd_id - prep_tab = prep_id + top_tab = "prep_template_tab" + sub_tab = prep_id + prep_tab = None else: top_tab = "preprocessed_data_tab" sub_tab = ppd_id @@ -604,12 +579,11 @@ def remove_add_study_template(self, raw_data, study_id, fp_rsp): Study(study_id)) remove(fp_rsp) - def remove_add_prep_template(self, fp_rpt, raw_data_id, study, - data_type_id, investigation_type): + def remove_add_prep_template(self, fp_rpt, study, data_type_id, + investigation_type): """add prep templates""" pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt), - RawData(raw_data_id), study, - _to_int(data_type_id), + study, _to_int(data_type_id), investigation_type=investigation_type).id remove(fp_rpt) return pt_id @@ -730,21 +704,18 @@ def delete_raw_data(self, study, user, callback): is done """ raw_data_id = int(self.get_argument('raw_data_id')) + prep_template_id = int(self.get_argument('prep_template_id')) try: - RawData.delete(raw_data_id, study.id) - msg = ("Raw data %d has been deleted from study: " - "%s" % (raw_data_id, study.title)) + RawData.delete(raw_data_id, prep_template_id) + msg = ("Raw data %d has been deleted from prep_template %d" + % (raw_data_id, prep_template_id)) msg_level = "success" - tab = 'study_information_tab' - tab_id = None except Exception as e: - msg = "Couldn't remove %d raw data: %s" % (raw_data_id, str(e)) + msg = "Couldn't remove raw data %d: %s" % (raw_data_id, str(e)) msg_level = "danger" - tab = 'raw_data_tab' - tab_id = raw_data_id - callback((msg, msg_level, tab, tab_id, None)) + callback((msg, msg_level, "prep_template_tab", prep_template_id, None)) def delete_prep_template(self, study, user, callback): """Delete the selected prep template @@ -771,7 +742,7 @@ def delete_prep_template(self, study, user, callback): msg = ("Couldn't remove prep template: %s" % str(e)) msg_level = "danger" - callback((msg, msg_level, 'raw_data_tab', prep_id, None)) + callback((msg, msg_level, 'prep_template_tab', prep_id, None)) def delete_preprocessed_data(self, study, user, callback): """Delete the selected preprocessed data @@ -850,7 +821,7 @@ def post(self, study_id): process_sample_template=self.process_sample_template, update_sample_template=self.update_sample_template, extend_sample_template=self.add_to_sample_template, - create_raw_data=self.create_raw_data, + add_raw_data=self.add_raw_data, add_prep_template=self.add_prep_template, make_public=self.make_public, approve_study=self.approve_study, diff --git a/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py b/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py index 249f617b7..7d05dd056 100644 --- a/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py +++ b/qiita_pet/handlers/study_handlers/metadata_summary_handlers.py @@ -62,8 +62,8 @@ def get(self, arguments): # The prep template has been provided template = self._get_template(PrepTemplate, prep_template) back_button_path = ( - "/study/description/%s?top_tab=raw_data_tab&sub_tab=%s" - "&prep_tab=%s" % (study_id, template.raw_data, template.id)) + "/study/description/%s?top_tab=prep_template_tab&sub_tab=%s" + % (study_id, template.id)) elif sample_template: # The sample template has been provided template = self._get_template(SampleTemplate, sample_template) diff --git a/qiita_pet/templates/study_description.html b/qiita_pet/templates/study_description.html index d8f8f0de1..a9f564b45 100644 --- a/qiita_pet/templates/study_description.html +++ b/qiita_pet/templates/study_description.html @@ -111,71 +111,158 @@ form.submit(); } -function create_raw_data() { - if ($("#previous_raw_data").val() === null && $("#filetype").val() === "") { - bootstrapAlert("You need to select either a new file type or a raw data used in a previous study."); - } else if ((typeof $("#previous_raw_data").val() != 'undefined' && $("#previous_raw_data").val() !== null) && $("#filetype").val() !== "") { - bootstrapAlert("You can only select a new file type or one used in a previous study but not both."); - } else { - if ($("#filetype").val() !== "") { - var form = $("
") - .attr("action", window.location.href) - .attr("method", "post") - .append($("") - .attr("type", "hidden") - .attr("name", "filetype") - .attr("value", $("#filetype").val())) - .append($("") - .attr("type", "hidden") - .attr("name", "action") - .attr("value", "create_raw_data")); - } else { - var form = $("") - .attr("action", window.location.href) - .attr("method", "post") - .append($("") - .attr("type", "hidden") - .attr("name", "previous_raw_data") - .attr("value", $("#previous_raw_data option:selected").map(function(){ return this.value }).get().join(","))) - .append($("") - .attr("type", "hidden") - .attr("name", "action") - .attr("value", "create_raw_data")); +function get_and_check_raw_data_files(prep_id, filetype){ + var barcodes = []; + var forward = []; + var reverse = []; + var sff = []; + var fasta = []; + var qual = []; + + var all_files = document.getElementsByName("upload_file_" + prep_id + "_" + filetype); + + for (var i = 0; i < all_files.length; i++){ + ele = all_files[i] + switch (ele.options[ele.selectedIndex].value) { + case "barcodes": + barcodes.push(ele.id); + break; + case "forward seqs": + forward.push(ele.id); + break; + case "reverse seqs": + reverse.push(ele.id); + break; + case "sff": + sff.push(ele.id); + break; + case "fasta": + fasta.push(ele.id); + break; + case "qual": + qual.push(ele.id); + break; } + } + + if (sff.length === 0 && barcodes.length === 0 && forward.length === 0 && fasta.length === 0) { + bootstrapAlert("You need to select at least: one barcode and one forward file, one sff file or one fasta file."); + return [false]; + } else if (sff.length === 0 && barcodes.length != forward.length) { + bootstrapAlert("You must select the same number of barcode and forward seq files."); + return [false]; + } else if (sff.length === 0 && (reverse.length !== 0 && reverse.length != forward.length)) { + bootstrapAlert("If you select reverse seqs, they should have the same number than barcodes and forward files."); + return [false]; + } else if (sff.length === 0 && (qual.length !== 0 && fasta.length != qual.length)) { + bootstrapAlert("If you select qual, you must have the same number of fasta and qual files."); + return [false]; + } else { + return [true, barcodes, forward, reverse, sff, fasta, qual]; + } +}; + +function create_raw_data(prep_id, filetype){ + var result = get_and_check_raw_data_files(prep_id, filetype); + var success = result[0]; + + if (success){ + var barcodes = result[1]; + var forward = result[2]; + var reverse = result[3]; + var sff = result[4]; + var fasta = result[5]; + var qual = result[6]; + + var form = $("") + .attr("action", "/study/create_raw_data") + .attr("method", "post") + .append($("") + .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "filetype") + .attr("value", filetype)) + .append($("") + .attr("type", "hidden") + .attr("name", "barcodes") + .attr("value", barcodes.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "forward") + .attr("value", forward.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "sff") + .attr("value", sff.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "fasta") + .attr("value", fasta.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "qual") + .attr("value", qual.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "reverse") + .attr("value", reverse.join())); $("body").append(form); form.submit(); + } +} +function add_raw_data(prep_id) { + raw_data_id = $("#previous-raw-data-" + prep_id).val(); + if (raw_data_id === null){ + bootstrapAlert("You need to select a raw data used in a previous study"); + } else { + var form = $("") + .attr("action", window.location.href) + .attr("method", "post") + .append($("") + .attr("type", "hidden") + .attr("name", "raw_data_id") + .attr("value", raw_data_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "action") + .attr("value", "add_raw_data")); + $("body").append(form); + form.submit(); } } -function add_prep_template(raw_data_id) { +function add_prep_template() { var form = $("") .attr("action", window.location.href) .attr("method", "post") - .append($("") - .attr("type", "hidden") - .attr("name", "raw_data_id") - .attr("value", raw_data_id)) .append($("") .attr("type", "hidden") .attr("name", "prep_template") - .attr("value", $("#add_prep_template_" + raw_data_id).val())) + .attr("value", $("#add_prep_template").val())) .append($("") .attr("type", "hidden") .attr("name", "investigation-type") - .attr("value", $("#investigation-type-" + raw_data_id).val())) + .attr("value", $("#investigation-type-new").val())) .append($("") .attr("type", "hidden") .attr("name", "user-defined-investigation-type") - .attr("value", $("#user-defined-investigation-type-" + raw_data_id).val())) + .attr("value", $("#user-defined-investigation-type-new").val())) .append($("") .attr("type", "hidden") .attr("name", "new-investigation-type") - .attr("value", $("#new-investigation-type-" + raw_data_id).val())) + .attr("value", $("#new-investigation-type-new").val())) .append($("") .attr("type", "hidden") .attr("name", "data_type_id") - .attr("value", $("#data_type_" + raw_data_id).val())) + .attr("value", $("#data_type").val())) .append($("") .attr("type", "hidden") .attr("name", "action") @@ -233,7 +320,7 @@ } } -function delete_raw_data(raw_data_filetype, raw_data_id) { +function delete_raw_data(raw_data_filetype, raw_data_id, prep_template_id) { if (confirm('Are you sure you want to delete raw data: ' + raw_data_filetype + ' (ID: ' + raw_data_id + ')?')) { var form = $("") .attr("action", window.location.href) @@ -244,6 +331,10 @@ .attr("value", raw_data_id)) .append($("") .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_template_id)) + .append($("") + .attr("type", "hidden") .attr("name", "action") .attr("value", "delete_raw_data")); $("body").append(form); @@ -419,90 +510,64 @@ } } -function link_files_to_raw_data(raw_data_id) { - var barcodes = []; - var forward = []; - var reverse = []; - var sff = []; - var fasta = []; - var qual = []; +function link_files_to_raw_data(raw_data_id, prep_template_id, filetype) { + var result = get_and_check_raw_data_files(prep_template_id, filetype); + var success = result[0]; - var all_files = document.getElementsByName("upload_file_" + raw_data_id); + if (success){ + var barcodes = result[1]; + var forward = result[2]; + var reverse = result[3]; + var sff = result[4]; + var fasta = result[5]; + var qual = result[6]; - for (var i = 0; i") - .attr("action", "/study/add_files_to_raw_data") - .attr("method", "post") - .append($("") - .attr("type", "hidden") - .attr("name", "raw_data_id") - .attr("value", raw_data_id)) - .append($("") - .attr("type", "hidden") - .attr("name", "study_id") - .attr("value", {{study.id}})) - .append($("") - .attr("type", "hidden") - .attr("name", "barcodes") - .attr("value", barcodes.join())) - .append($("") - .attr("type", "hidden") - .attr("name", "forward") - .attr("value", forward.join())) - .append($("") - .attr("type", "hidden") - .attr("name", "sff") - .attr("value", sff.join())) - .append($("") - .attr("type", "hidden") - .attr("name", "fasta") - .attr("value", fasta.join())) - .append($("") - .attr("type", "hidden") - .attr("name", "qual") - .attr("value", qual.join())) - .append($("") - .attr("type", "hidden") - .attr("name", "reverse") - .attr("value", reverse.join())); + .attr("action", "/study/add_files_to_raw_data") + .attr("method", "post") + .append($("") + .attr("type", "hidden") + .attr("name", "raw_data_id") + .attr("value", raw_data_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_template_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "study_id") + .attr("value", {{study.id}})) + .append($("") + .attr("type", "hidden") + .attr("name", "barcodes") + .attr("value", barcodes.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "forward") + .attr("value", forward.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "sff") + .attr("value", sff.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "fasta") + .attr("value", fasta.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "qual") + .attr("value", qual.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "reverse") + .attr("value", reverse.join())); $("body").append(form); form.submit(); } } -function unlink_all_files(raw_data_id) { - if (confirm("Are you sure you want to unlink all files? They will be removed from the system")){ +function unlink_all_files(raw_data_id, prep_template_id) { + if (confirm("Are you sure you want to unlink all files?")){ var form = $("") .attr("action", "/study/unlink_all_files") .attr("method", "post") @@ -514,6 +579,10 @@ .attr("type", "hidden") .attr("name", "raw_data_id") .attr("value", raw_data_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_template_id)) $("body").append(form); form.submit(); } @@ -529,9 +598,9 @@ function display_tab(top_tab, sub_tab, prep_tab){ $('#myTab a[href="#'+top_tab+'"]').tab('show'); - if (top_tab == "raw_data_tab"){ + if (top_tab == "prep_template_tab"){ if (sub_tab !== undefined){ - $('#raw_data_nav_tabs a[href="#raw_data_info_'+sub_tab+'"]').tab('show') + $('#prep_template_nav_tabs a[href="#prep_template_info_'+sub_tab+'"]').tab('show') } } else if (top_tab == "preprocessed_data_tab"){ @@ -555,6 +624,7 @@ , "{{prep_tab}}" {% end %} {% end %}) + $('.filesgroup').hide(); }); @@ -591,7 +661,7 @@

{{study_alias}}

Study information {% if show_data_tabs %} -
  • Raw data
  • +
  • Prep templates
  • Preprocessed data
  • Processed data
  • {% end %} @@ -603,7 +673,7 @@

    {{study_alias}}

    {% module StudyInformationTab(study) %} {% if show_data_tabs %} - {% module RawDataTab(study, full_access) %} + {% module PrepTemplateTab(study, full_access) %} {% module PreprocessedDataTab(study, full_access) %} diff --git a/qiita_pet/templates/study_description_templates/prep_template_info_tab.html b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html new file mode 100644 index 000000000..52ede428f --- /dev/null +++ b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html @@ -0,0 +1,192 @@ + + + + +
    + + + + + + + + + + {% if raw_data %} + + + + + + + + {% else %} + + + + + {% end %} +
    + {% if is_editable %} + {% module EditInvestigationType(ena_terms, user_defined_terms, pt_id, investigation_type, 0) %} + {% else %} + Investigation type: {{investigation_type}} + {% end %} +
    + + Show prep template summary + +
    {% raw current_template_fp %}
    {% raw current_qiime_fp %}
    {% module RawDataInfoDiv(raw_data, prep_template, study, files) %}
    + {% if not preprocessed_data or preprocessing_status.startswith('failed') %} + {% if preprocessing_status == 'not_preprocessed' or preprocessing_status.startswith('failed')%} + {% if show_preprocess_btn %} + Preprocess + {% else %} + {{ no_preprocess_msg }} + {% end %} + {% end %} +
    + Status: {{preprocessing_status}} +
    + {% if 'zero-size array' in preprocessing_status %} + Your barcodes do not seem to match your prep template info + {% end %} + {% else %} + Preprocessed data generated: + {{preprocessed_data[0]}} + {% for pdi in preprocessed_data[1:] %} + , {{pdi}} + {% end %} + {% end %} +
    +
    + +
    + +
    +
    + What file type is your raw data? + +
    + {% for _, value, filepath_types in filetypes %} +
    +
    + Choose the files to link to the raw data +
    + + + + + + + {% for f in files %} + + + + + + {% end %} +
    File   File type
    {{f}}  + +
    + Link +
    + {% end %} +
    +
    +
    + +
    + +
    +
    + {% if other_studies_rd %} + Choose raw data from studies that you have access to: +
    + +
    +
    + Add + {% else %} + There is no existent raw data available. + {% end %} +
    +
    +
    +
    +
    +
    diff --git a/qiita_pet/templates/study_description_templates/prep_template_tab.html b/qiita_pet/templates/study_description_templates/prep_template_tab.html new file mode 100644 index 000000000..0f8f7a038 --- /dev/null +++ b/qiita_pet/templates/study_description_templates/prep_template_tab.html @@ -0,0 +1,83 @@ +
    + + + + +
    + +
    + To add a prep template you need to: +
      + +
    1. + Select your prep template file (only files with the "txt" and "tsv" file extensions will be displayed here): + +
    2. + +
    3. + Select the prep template data type + +
    4. + +
    5. + Select an investigation type (optional but required for EBI submission): + + + + + + + + + + +
      +
    6. +
    +
    + Add prep template +
    + + + {% for _, _, pt, _ in available_prep_templates %} + {% module PrepTemplateInfoTab(study, pt, full_access, ena_terms, user_defined_terms) %} + {% end %} + +
    + +
    diff --git a/qiita_pet/templates/study_description_templates/preprocessed_data_info_tab.html b/qiita_pet/templates/study_description_templates/preprocessed_data_info_tab.html index f36d7454b..17669f0f0 100644 --- a/qiita_pet/templates/study_description_templates/preprocessed_data_info_tab.html +++ b/qiita_pet/templates/study_description_templates/preprocessed_data_info_tab.html @@ -78,7 +78,7 @@
    - Prep template
    + Prep template
    EBI status: {{ebi_status}}
    EBI study accession: {{ebi_study_accession}}
    EBI submission accession: {{ebi_submission_accession}}
    diff --git a/qiita_pet/templates/study_description_templates/raw_data_info.html b/qiita_pet/templates/study_description_templates/raw_data_info.html new file mode 100644 index 000000000..163e3f179 --- /dev/null +++ b/qiita_pet/templates/study_description_templates/raw_data_info.html @@ -0,0 +1,56 @@ +
    +
    +
    + +
    +
    + {% if raw_data_files %} +

    Files linked to this raw data:


    + {% for fname, ftype in raw_data_files %} + {{fname}}: {{ftype}}
    + {% end %} + {% if show_buttons %} +
    + Unlink all files
    + {% end %} + {% else %} +

    There are no files currently linked to the raw data. Choose from the following list:


    + + + + + + + {% for f in files %} + + + + + + {% end %} + {% if show_buttons %} + + + {% end %} +
    File   File type
    {{f}}  + +
     
    Link files
    + {% end %} + {{link_msg}}
    +
    +
    +
    +
    +
    \ No newline at end of file diff --git a/qiita_pet/templates/study_description_templates/raw_data_tab.html b/qiita_pet/templates/study_description_templates/raw_data_tab.html deleted file mode 100644 index e8c6e3b3e..000000000 --- a/qiita_pet/templates/study_description_templates/raw_data_tab.html +++ /dev/null @@ -1,50 +0,0 @@ -
    - - - - -
    - -
    - What file type is your raw data? - - {% if other_studies_rd %} -

    - You could also select raw data from other studies that you have access to: -
    - - {% end %} -

    - Add raw data -
    - - - {% for _, _, rd, _ in available_raw_data %} - {% module RawDataEditorTab(study, rd, full_access) %} - {% end %} - -
    - -
    diff --git a/qiita_pet/test/test_study_handlers.py b/qiita_pet/test/test_study_handlers.py index d5a2bc4eb..b289d4287 100644 --- a/qiita_pet/test/test_study_handlers.py +++ b/qiita_pet/test/test_study_handlers.py @@ -56,7 +56,7 @@ def setUp(self): 'metadata_complete': True, 'study_title': 'Identification of the Microbiomes for Cannabis Soils', - 'num_raw_data': 4, + 'num_raw_data': 1, 'number_samples_collected': 27, 'shared': 'Shared', @@ -201,31 +201,6 @@ def test_update_sample_template(self): response = self.post('/study/description/1', post_args) self.assertEqual(response.code, 200) - def test_create_raw_data(self): - # testing adding new raw data - post_args = { - 'filetype': '1', - 'action': 'create_raw_data' - } - response = self.post('/study/description/1', post_args) - self.assertEqual(response.code, 200) - - # testing an error due to previous raw data already added - post_args = { - 'previous_raw_data': '1', - 'action': 'create_raw_data' - } - response = self.post('/study/description/1', post_args) - self.assertEqual(response.code, 500) - - # testing an error due to previous_raw_data not existing - post_args = { - 'previous_raw_data': '5', - 'action': 'create_raw_data' - } - response = self.post('/study/description/1', post_args) - self.assertEqual(response.code, 500) - class TestStudyEditHandler(TestHandlerBase): database = True @@ -360,7 +335,7 @@ class TestSearchStudiesAJAX(TestHandlerBase): 'pmid': '123456, 7891011', - 'num_raw_data': 4, + 'num_raw_data': 1, 'proc_data_info': [{ 'pid': 1, 'processed_date': '2012-10-01 09:30:27', @@ -484,12 +459,13 @@ def test_delete_sample_template(self): def test_delete_raw_data(self): response = self.post('/study/description/1', {'raw_data_id': 1, + 'prep_template_id': 1, 'action': 'delete_raw_data'}) self.assertEqual(response.code, 200) # checking that the action was sent - self.assertIn("Raw data 1 has prep template(s) associated so it can't " - "be erased", response.body) + self.assertIn("Couldn't remove raw data 1: Raw data (1) can't be " + "remove because it has linked files", response.body) def test_delete_prep_template(self): response = self.post('/study/description/1', diff --git a/qiita_pet/uimodules/__init__.py b/qiita_pet/uimodules/__init__.py index 88f893074..43fcf9724 100644 --- a/qiita_pet/uimodules/__init__.py +++ b/qiita_pet/uimodules/__init__.py @@ -7,12 +7,12 @@ # ----------------------------------------------------------------------------- from .study_information_tab import StudyInformationTab -from .raw_data_tab import (RawDataTab, RawDataEditorTab, PrepTemplatePanel, - EditInvestigationType) +from .prep_template_tab import (PrepTemplateTab, PrepTemplateInfoTab, + RawDataInfoDiv, EditInvestigationType) from .preprocessed_data_tab import PreprocessedDataTab, PreprocessedDataInfoTab from .processed_data_tab import ProcessedDataTab, ProcessedDataInfoTab -__all__ = ['StudyInformationTab', 'RawDataTab', 'RawDataEditorTab', - 'PrepTemplatePanel', 'EditInvestigationType', 'PreprocessedDataTab', +__all__ = ['StudyInformationTab', 'PrepTemplateTab', 'PrepTemplateInfoTab', + 'EditInvestigationType', 'RawDataInfoDiv', 'PreprocessedDataTab', 'PreprocessedDataInfoTab', 'ProcessedDataTab', 'ProcessedDataInfoTab'] diff --git a/qiita_pet/uimodules/prep_template_tab.py b/qiita_pet/uimodules/prep_template_tab.py new file mode 100644 index 000000000..2686ed295 --- /dev/null +++ b/qiita_pet/uimodules/prep_template_tab.py @@ -0,0 +1,274 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from operator import itemgetter +from os.path import basename +from collections import defaultdict + +from future.utils import viewitems + +from qiita_db.util import (get_filetypes, get_files_from_uploads_folders, + get_data_types, convert_to_id, get_filepath_types) +from qiita_db.study import Study +from qiita_db.data import RawData +from qiita_db.ontology import Ontology +from qiita_db.metadata_template import (PrepTemplate, TARGET_GENE_DATA_TYPES, + PREP_TEMPLATE_COLUMNS_TARGET_GENE) +from qiita_db.parameters import (Preprocessed454Params, + PreprocessedIlluminaParams) +from qiita_pet.util import STATUS_STYLER +from qiita_pet.handlers.util import download_link_or_path +from .base_uimodule import BaseUIModule + + +filepath_types = [k.split('_', 1)[1].replace('_', ' ') + for k in get_filepath_types() + if k.startswith('raw_')] +fp_type_by_ft = defaultdict( + lambda: filepath_types, SFF=['sff'], FASTA=['fasta', 'qual'], + FASTQ=['barcodes', 'forward seqs', 'reverse seqs']) + + +def _get_accessible_raw_data(user): + """Retrieves a tuple of raw_data_id and the last study title for that + raw_data + """ + d = {} + for sid in user.user_studies: + for rdid in Study(sid).raw_data(): + d[int(rdid)] = Study(RawData(rdid).studies[-1]).title + return d + + +def _template_generator(study, full_access): + """Generates tuples of prep template information + + Parameters + ---------- + study : Study + The study to get all the prep templates + full_access : boolean + A boolean that indicates if the user has full access to the study + + Returns + ------- + Generator of tuples of (int, str, PrepTemplate, (str, str, str)) + Each tuple contains the prep template id, the prep template data_type + the PrepTemplate object and a tuple with 3 strings for the style of + the prep template status icons + """ + for pt_id in study.prep_templates(): + pt = PrepTemplate(pt_id) + if full_access or pt.status() == 'public': + yield (pt.id, pt.data_type(), pt, STATUS_STYLER[pt.status]) + + +class PrepTemplateTab(BaseUIModule): + def render(self, study, full_access): + files = [f for _, f in get_files_from_uploads_folders(str(study.id))] + data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) + prep_templates_info = [ + res for res in _template_generator(study, full_access)] + # Get all the ENA terms for the investigation type + ontology = Ontology(convert_to_id('ENA', 'ontology')) + # make "Other" show at the bottom of the drop down menu + ena_terms = [] + for v in sorted(ontology.terms): + if v != 'Other': + ena_terms.append('' % (v, v)) + ena_terms.append('') + + # New Type is for users to add a new user-defined investigation type + user_defined_terms = ontology.user_defined_terms + ['New Type'] + + return self.render_string( + "study_description_templates/prep_template_tab.html", + files=files, + data_types=data_types, + available_prep_templates=prep_templates_info, + ena_terms=ena_terms, + user_defined_terms=user_defined_terms, + study=study, + full_access=full_access) + + +class PrepTemplateInfoTab(BaseUIModule): + def render(self, study, prep_template, full_access, ena_terms, + user_defined_terms): + user = self.current_user + is_local_request = self._is_local() + + template_fps = [] + qiime_fps = [] + # Unfortunately, both the prep template and the qiime mapping files + # have the sample type. The way to differentiate them is if we have + # the substring 'qiime' in the basename + for id_, fp in prep_template.get_filepaths(): + if 'qiime' in basename(fp): + qiime_fps.append( + download_link_or_path( + is_local_request, fp, id_, 'Qiime mapping')) + else: + template_fps.append( + download_link_or_path( + is_local_request, fp, id_, 'Prep template')) + + # Since get_filepaths returns the paths sorted from newest to oldest, + # the first in both list is the latest one + current_template_fp = template_fps[0] + current_qiime_fp = qiime_fps[0] + + if len(template_fps) > 1: + show_old_templates = True + old_templates = template_fps[1:] + else: + show_old_templates = False + old_templates = None + + if len(qiime_fps) > 1: + show_old_qiime_fps = True + old_qiime_fps = qiime_fps[1:] + else: + show_old_qiime_fps = False + old_qiime_fps = None + + filetypes = sorted( + ((ft, ft_id, fp_type_by_ft[ft]) + for ft, ft_id in viewitems(get_filetypes())), + key=itemgetter(1)) + files = [f for _, f in get_files_from_uploads_folders(str(study.id))] + + other_studies_rd = sorted(viewitems( + _get_accessible_raw_data(user))) + + # A prep template can be modified if its status is sanbdox + is_editable = prep_template.status == 'sanbdox' + + raw_data_id = prep_template.raw_data + preprocess_options = [] + preprocessed_data = None + show_preprocess_btn = True + no_preprocess_msg = None + if raw_data_id: + rd = RawData(raw_data_id) + rd_ft = rd.filetype + # If the prep template has a raw data associated, it can be + # preprocessed. Retrieve the pre-processing parameters + if rd_ft in ('SFF', 'FASTA'): + param_iter = Preprocessed454Params.iter() + elif rd_ft == 'FASTQ': + param_iter = PreprocessedIlluminaParams.iter() + else: + raise NotImplementedError( + "Pre-processing of %s files currently not supported." + % rd_ft) + + preprocess_options = [] + for param in param_iter: + text = ("%s: %s" % (k, v) + for k, v in viewitems(param.values)) + preprocess_options.append((param.id, + param.name, + '
    '.join(text))) + preprocessed_data = prep_template.preprocessed_data + + # Check if the template have all the required columns for + # preprocessing + raw_data_files = rd.get_filepaths() + if len(raw_data_files) == 0: + show_preprocess_btn = False + no_preprocess_msg = ( + "Preprocessing disabled because there are no files " + "linked with the Raw Data") + else: + if prep_template.data_type() in TARGET_GENE_DATA_TYPES: + key = ('demultiplex_multiple' if len(raw_data_files) > 2 + else 'demultiplex') + missing_cols = prep_template.check_restrictions( + [PREP_TEMPLATE_COLUMNS_TARGET_GENE[key]]) + show_preprocess_btn = len(missing_cols) == 0 + if not show_preprocess_btn: + no_preprocess_msg = ( + "Preprocessing disabled due to missing columns in " + "the prep template: %s" % ', '.join(missing_cols)) + else: + no_preprocess_msg = None + + preprocessing_status = prep_template.preprocessing_status + + return self.render_string( + "study_description_templates/prep_template_info_tab.html", + pt_id=prep_template.id, + study_id=study.id, + raw_data=raw_data_id, + current_template_fp=current_template_fp, + current_qiime_fp=current_qiime_fp, + show_old_templates=show_old_templates, + old_templates=old_templates, + show_old_qiime_fps=show_old_qiime_fps, + old_qiime_fps=old_qiime_fps, + filetypes=filetypes, + files=files, + other_studies_rd=other_studies_rd, + prep_template=prep_template, + study=study, + ena_terms=ena_terms, + user_defined_terms=user_defined_terms, + investigation_type=prep_template.investigation_type, + is_editable=is_editable, + preprocess_options=preprocess_options, + preprocessed_data=preprocessed_data, + preprocessing_status=preprocessing_status, + show_preprocess_btn=show_preprocess_btn, + no_preprocess_msg=no_preprocess_msg) + + +class RawDataInfoDiv(BaseUIModule): + def render(self, raw_data_id, prep_template, study, files): + rd = RawData(raw_data_id) + raw_data_files = [(basename(fp), fp_type[4:]) + for _, fp, fp_type in rd.get_filepaths()] + filetype = rd.filetype + fp_types = fp_type_by_ft[filetype] + raw_data_link_status = rd.link_filepaths_status + + show_buttons = rd.status(study) == 'sandbox' + link_msg = "" + if show_buttons: + # Define the message for the link status + if raw_data_link_status == 'linking': + link_msg = "Linking files..." + show_buttons = False + elif raw_data_link_status == 'unlinking': + link_msg = "Unlinking files..." + show_buttons = False + elif raw_data_link_status.startswith('failed'): + link_msg = "Error (un)linking files: %s" % raw_data_link_status + + return self.render_string( + "study_description_templates/raw_data_info.html", + rd_id=raw_data_id, + rd_filetype=rd.filetype, + raw_data_files=raw_data_files, + prep_template_id=prep_template.id, + files=files, + filepath_types=fp_types, + filetype=filetype, + link_msg=link_msg, + show_buttons=show_buttons) + + +class EditInvestigationType(BaseUIModule): + def render(self, ena_terms, user_defined_terms, prep_id, inv_type, ppd_id): + return self.render_string( + "study_description_templates/edit_investigation_type.html", + ena_terms=ena_terms, + user_defined_terms=user_defined_terms, + prep_id=prep_id, + investigation_type=inv_type, + ppd_id=ppd_id) diff --git a/qiita_pet/uimodules/raw_data_tab.py b/qiita_pet/uimodules/raw_data_tab.py deleted file mode 100644 index 495309bd3..000000000 --- a/qiita_pet/uimodules/raw_data_tab.py +++ /dev/null @@ -1,254 +0,0 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) 2014--, The Qiita Development Team. -# -# Distributed under the terms of the BSD 3-clause License. -# -# The full license is in the file LICENSE, distributed with this software. -# ----------------------------------------------------------------------------- - -from operator import itemgetter -from os.path import basename - -from future.utils import viewitems - -from qiita_db.util import (get_filetypes, get_files_from_uploads_folders, - get_data_types, convert_to_id, get_filepath_types) -from qiita_db.study import Study -from qiita_db.data import RawData -from qiita_db.ontology import Ontology -from qiita_db.metadata_template import (PrepTemplate, TARGET_GENE_DATA_TYPES, - PREP_TEMPLATE_COLUMNS_TARGET_GENE) -from qiita_db.parameters import (Preprocessed454Params, - PreprocessedIlluminaParams) -from qiita_pet.util import STATUS_STYLER -from .base_uimodule import BaseUIModule - - -def get_raw_data_from_other_studies(user, study): - """Retrieves a tuple of raw_data_id and the last study title for that - raw_data - """ - d = {} - for sid in user.user_studies: - if sid == study.id: - continue - for rdid in Study(sid).raw_data(): - d[int(rdid)] = Study(RawData(rdid).studies[-1]).title - return d - - -def get_raw_data(rdis): - """Get all raw data objects from a list of raw_data_ids""" - return [RawData(rdi) for rdi in rdis] - - -class RawDataTab(BaseUIModule): - def render(self, study, full_access): - user = self.current_user - - filetypes = sorted(viewitems(get_filetypes()), key=itemgetter(1)) - other_studies_rd = sorted(viewitems( - get_raw_data_from_other_studies(user, study))) - - raw_data_info = [ - (rd.id, rd.filetype, rd, STATUS_STYLER[rd.status(study)]) - for rd in get_raw_data(study.raw_data()) - if full_access or rd.status(study) == 'public'] - - return self.render_string( - "study_description_templates/raw_data_tab.html", - filetypes=filetypes, - other_studies_rd=other_studies_rd, - available_raw_data=raw_data_info, - study=study, - full_access=full_access) - - -class RawDataEditorTab(BaseUIModule): - def render(self, study, raw_data, full_access): - user = self.current_user - study_status = study.status - user_level = user.level - raw_data_id = raw_data.id - files = [f for _, f in get_files_from_uploads_folders(str(study.id))] - - # Get the available prep template data types - data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) - - # Get all the ENA terms for the investigation type - ontology = Ontology(convert_to_id('ENA', 'ontology')) - # make "Other" show at the bottom of the drop down menu - ena_terms = [] - for v in sorted(ontology.terms): - if v != 'Other': - ena_terms.append('' % (v, v)) - ena_terms.append('') - - # New Type is for users to add a new user-defined investigation type - user_defined_terms = ontology.user_defined_terms + ['New Type'] - - # Get all the information about the prep templates - available_prep_templates = [] - for p in sorted(raw_data.prep_templates): - if PrepTemplate.exists(p): - pt = PrepTemplate(p) - # if the prep template doesn't belong to this study, skip - if (study.id == pt.study_id and - (full_access or pt.status == 'public')): - available_prep_templates.append(pt) - - # getting filepath_types - if raw_data.filetype == 'SFF': - fts = ['sff'] - elif raw_data.filetype == 'FASTA': - fts = ['fasta', 'qual'] - elif raw_data.filetype == 'FASTQ': - fts = ['barcodes', 'forward seqs', 'reverse seqs'] - else: - fts = [k.split('_', 1)[1].replace('_', ' ') - for k in get_filepath_types() if k.startswith('raw_')] - - # The raw data can be edited (e.i. adding prep templates and files) - # only if the study is sandboxed or the current user is an admin - is_editable = study_status == 'sandbox' or user_level == 'admin' - - # Get the files linked with the raw_data - raw_data_files = raw_data.get_filepaths() - - # Get the status of the data linking - raw_data_link_status = raw_data.link_filepaths_status - - # By default don't show the unlink button - show_unlink_btn = False - # By default disable the the link file button - disable_link_btn = True - # Define the message for the link status - if raw_data_link_status == 'linking': - link_msg = "Linking files..." - elif raw_data_link_status == 'unlinking': - link_msg = "Unlinking files..." - else: - # The link button is only disable if raw data link status is - # linking or unlinking, so we can enable it here - disable_link_btn = False - # The unlink button is only shown if the study is editable, the raw - # data linking status is not in linking or unlinking, and there are - # files attached to the raw data. At this point, we are sure that - # the raw data linking status is not in linking or unlinking so we - # still need to check if it is editable or there are files attached - show_unlink_btn = is_editable and raw_data_files - if raw_data_link_status.startswith('failed'): - link_msg = "Error (un)linking files: %s" % raw_data_link_status - else: - link_msg = "" - - # Get the raw_data filetype - raw_data_filetype = raw_data.filetype - - return self.render_string( - "study_description_templates/raw_data_editor_tab.html", - study_id=study.id, - study_status=study_status, - user_level=user_level, - raw_data_id=raw_data_id, - files=files, - data_types=data_types, - ena_terms=ena_terms, - user_defined_terms=user_defined_terms, - available_prep_templates=available_prep_templates, - filepath_types=fts, - is_editable=is_editable, - show_unlink_btn=show_unlink_btn, - link_msg=link_msg, - raw_data_files=raw_data_files, - raw_data_filetype=raw_data_filetype, - disable_link_btn=disable_link_btn) - - -class PrepTemplatePanel(BaseUIModule): - def render(self, prep, study_id, is_editable, ena_terms, - study_status, user_defined_terms, raw_data_files): - # Check if the request came from a local source - is_local_request = self._is_local() - - prep_id = prep.id - status_class1, status_class2, status_color = STATUS_STYLER[prep.status] - data_type = prep.data_type() - raw_data = RawData(prep.raw_data) - filepaths = prep.get_filepaths() - investigation_type = prep.investigation_type - preprocessed_data = prep.preprocessed_data - preprocessing_status = prep.preprocessing_status - - if raw_data.filetype in ('SFF', 'FASTA'): - param_iter = Preprocessed454Params.iter() - elif raw_data.filetype == 'FASTQ': - param_iter = PreprocessedIlluminaParams.iter() - else: - raise ValueError("Don't know what to do but this exception will " - "never actually get shown anywhere because why " - "would you want to see tracebacks?") - - preprocess_options = [] - for param in param_iter: - text = ("%s: %s" % (k, v) - for k, v in viewitems(param.values)) - preprocess_options.append((param.id, - param.name, - '
    '.join(text))) - - # Unfortunately, both the prep template and the qiime mapping files - # have the sample type. The way to differentiate them is if we have - # the substring 'qiime' in the basename - _fp_type = (lambda fp: "Qiime mapping" - if 'qiime' in basename(fp) else "Prep template") - filepaths = [(id_, fp, _fp_type(fp)) for id_, fp in filepaths] - - # Check if the template have all the required columns for preprocessing - if prep.data_type() in TARGET_GENE_DATA_TYPES: - key = ('demultiplex_multiple' if len(raw_data_files) > 2 - else 'demultiplex') - missing_cols = prep.check_restrictions( - [PREP_TEMPLATE_COLUMNS_TARGET_GENE[key]]) - show_preprocess_btn = len(missing_cols) == 0 - if not show_preprocess_btn: - no_preprocess_msg = ( - "Preprocessing disabled due to missing columns in the " - "prep template: %s" % ', '.join(missing_cols)) - else: - no_preprocess_msg = None - else: - show_preprocess_btn = True - no_preprocess_msg = None - - return self.render_string( - "study_description_templates/prep_template_panel.html", - prep_id=prep_id, - status_class1=status_class1, - status_class2=status_class2, - status_color=status_color, - data_type=data_type, - filepaths=filepaths, - investigation_type=investigation_type, - preprocessed_data=preprocessed_data, - preprocessing_status=preprocessing_status, - study_id=study_id, - is_local_request=is_local_request, - is_editable=is_editable, - ena_terms=ena_terms, - study_status=study_status, - user_defined_terms=user_defined_terms, - preprocess_options=preprocess_options, - show_preprocess_btn=show_preprocess_btn, - no_preprocess_msg=no_preprocess_msg) - - -class EditInvestigationType(BaseUIModule): - def render(self, ena_terms, user_defined_terms, prep_id, inv_type, ppd_id): - return self.render_string( - "study_description_templates/edit_investigation_type.html", - ena_terms=ena_terms, - user_defined_terms=user_defined_terms, - prep_id=prep_id, - investigation_type=inv_type, - ppd_id=ppd_id) diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 9097fe1d5..324c31b86 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -28,7 +28,7 @@ from qiita_pet.handlers.logger_handlers import LogEntryViewerHandler from qiita_pet.handlers.upload import UploadFileHandler, StudyUploadFileHandler from qiita_pet.handlers.compute import ( - ComputeCompleteHandler, AddFilesToRawData, UnlinkAllFiles) + ComputeCompleteHandler, AddFilesToRawData, UnlinkAllFiles, CreateRawData) from qiita_pet.handlers.preprocessing_handlers import PreprocessHandler from qiita_pet.handlers.processing_handlers import ProcessHandler from qiita_pet.handlers.stats import StatsHandler @@ -81,6 +81,7 @@ def __init__(self): (r"/study/list/socket/", SelectSamplesHandler), (r"/study/search/(.*)", SearchStudiesAJAX), (r"/study/add_files_to_raw_data", AddFilesToRawData), + (r"/study/create_raw_data", CreateRawData), (r"/study/unlink_all_files", UnlinkAllFiles), (r"/study/preprocess", PreprocessHandler), (r"/study/process", ProcessHandler), diff --git a/qiita_ware/dispatchable.py b/qiita_ware/dispatchable.py index 7ff497bee..0c5883541 100644 --- a/qiita_ware/dispatchable.py +++ b/qiita_ware/dispatchable.py @@ -72,6 +72,14 @@ def run_analysis(analysis_id, commands, comm_opts=None, return ar(analysis, commands, comm_opts, rarefaction_depth) +def create_raw_data(filetype, prep_template, filepaths): + """Creates a new raw data + + Needs to be dispachable because it moves large files + """ + RawData.create(filetype, [prep_template], filepaths) + + def add_files_to_raw_data(raw_data_id, filepaths): """Add files to raw data diff --git a/test_data_studies/commands.sh b/test_data_studies/commands.sh index 8cc709b54..779ac3e33 100755 --- a/test_data_studies/commands.sh +++ b/test_data_studies/commands.sh @@ -40,19 +40,19 @@ for i in ${studies[@]}; do qiita db load_sample_template $sample_file --study $study_id echo "Ok" + # Loading prep template + echo "\tloading prep template... " + output=`qiita db load_prep_template $prep_file --study $study_id --data_type "16S"` + pt_id=`echo -e "${output}" | cut -d " " -f 10` + echo "Ok" + # Loading raw data echo "\tloading raw data... " echo -e ">seq\nAAAA" > seqs.fna - output="`qiita db load_raw --fp seqs.fna --fp_type raw_forward_seqs --filetype FASTQ --study $i`" + output="`qiita db load_raw --fp seqs.fna --fp_type raw_forward_seqs --filetype FASTQ --prep_template $pt_id`" raw_id=`echo -e "${output}" | cut -d " " -f 10` echo "Ok" - # Loading prep template - echo "\tloading prep template... " - output=`qiita db load_prep_template $prep_file --raw_data $raw_id --study $study_id --data_type "16S"` - pt_id=`echo -e "${output}" | cut -d " " -f 10` - echo "Ok" - # Loading preprocessed data echo "\tloading preprocessed data... " echo -e ">seq\nAAAA" > temp/seqs.fna