From 36c01b0692c55f38e981cc6aa2214cd9e521fddf Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Fri, 22 May 2015 11:51:54 -0700 Subject: [PATCH 01/19] Change RawDataTab to PrepTemplateTab --- .../study_handlers/description_handlers.py | 20 ++--- qiita_pet/templates/study_description.html | 24 +++--- .../prep_template_tab.html | 83 +++++++++++++++++++ .../raw_data_tab.html | 50 ----------- qiita_pet/uimodules/__init__.py | 6 +- .../{raw_data_tab.py => prep_template_tab.py} | 51 +++++++++--- 6 files changed, 142 insertions(+), 92 deletions(-) create mode 100644 qiita_pet/templates/study_description_templates/prep_template_tab.html delete mode 100644 qiita_pet/templates/study_description_templates/raw_data_tab.html rename qiita_pet/uimodules/{raw_data_tab.py => prep_template_tab.py} (85%) diff --git a/qiita_pet/handlers/study_handlers/description_handlers.py b/qiita_pet/handlers/study_handlers/description_handlers.py index 4f3295c79..c49b86c1f 100644 --- a/qiita_pet/handlers/study_handlers/description_handlers.py +++ b/qiita_pet/handlers/study_handlers/description_handlers.py @@ -379,10 +379,9 @@ def add_prep_template(self, study, user, callback): msg = "Your prep template was added" msg_level = "success" - # If we are on this function, the arguments "raw_data_id", - # "prep_template" and "data_type_id" must be defined. If not, + # If we are on this function, the arguments "prep_template" and + # "data_type_id" must be defined. If not, # let tornado raise its error - raw_data_id = self.get_argument('raw_data_id') prep_template = self.get_argument('prep_template') data_type_id = self.get_argument('data_type_id') @@ -397,8 +396,6 @@ def add_prep_template(self, study, user, callback): investigation_type, user_defined_investigation_type, new_investigation_type) - # Make sure that the id is an integer - raw_data_id = _to_int(raw_data_id) # Get the upload base directory _, base_path = get_mountpoint("uploads")[0] # Get the path to the prep template @@ -413,8 +410,8 @@ def add_prep_template(self, study, user, callback): warnings.simplefilter("always") # deleting previous uploads and inserting new one - pt_id = self.remove_add_prep_template(fp_rpt, raw_data_id, - study, data_type_id, + pt_id = self.remove_add_prep_template(fp_rpt, study, + data_type_id, investigation_type) # join all the warning messages into one. Note that this info @@ -432,7 +429,7 @@ def add_prep_template(self, study, user, callback): basename(fp_rpt), str(e)) msg_level = "danger" - callback((msg, msg_level, 'raw_data_tab', raw_data_id, pt_id)) + callback((msg, msg_level, 'prep_template_tab', pt_id, None)) def make_public(self, study, user, callback): """Makes the current study public @@ -603,12 +600,11 @@ def remove_add_study_template(self, raw_data, study_id, fp_rsp): Study(study_id)) remove(fp_rsp) - def remove_add_prep_template(self, fp_rpt, raw_data_id, study, - data_type_id, investigation_type): + def remove_add_prep_template(self, fp_rpt, study, data_type_id, + investigation_type): """add prep templates""" pt_id = PrepTemplate.create(load_template_to_dataframe(fp_rpt), - RawData(raw_data_id), study, - _to_int(data_type_id), + study, _to_int(data_type_id), investigation_type=investigation_type).id remove(fp_rpt) return pt_id diff --git a/qiita_pet/templates/study_description.html b/qiita_pet/templates/study_description.html index 20f39ba01..3cd9ee4f2 100644 --- a/qiita_pet/templates/study_description.html +++ b/qiita_pet/templates/study_description.html @@ -148,34 +148,30 @@ } } -function add_prep_template(raw_data_id) { +function add_prep_template() { var form = $("
") .attr("action", window.location.href) .attr("method", "post") - .append($("") - .attr("type", "hidden") - .attr("name", "raw_data_id") - .attr("value", raw_data_id)) .append($("") .attr("type", "hidden") .attr("name", "prep_template") - .attr("value", $("#add_prep_template_" + raw_data_id).val())) + .attr("value", $("#add_prep_template").val())) .append($("") .attr("type", "hidden") .attr("name", "investigation-type") - .attr("value", $("#investigation-type-" + raw_data_id).val())) + .attr("value", $("#investigation-type-new").val())) .append($("") .attr("type", "hidden") .attr("name", "user-defined-investigation-type") - .attr("value", $("#user-defined-investigation-type-" + raw_data_id).val())) + .attr("value", $("#user-defined-investigation-type-new").val())) .append($("") .attr("type", "hidden") .attr("name", "new-investigation-type") - .attr("value", $("#new-investigation-type-" + raw_data_id).val())) + .attr("value", $("#new-investigation-type-new").val())) .append($("") .attr("type", "hidden") .attr("name", "data_type_id") - .attr("value", $("#data_type_" + raw_data_id).val())) + .attr("value", $("#data_type").val())) .append($("") .attr("type", "hidden") .attr("name", "action") @@ -529,9 +525,9 @@ function display_tab(top_tab, sub_tab, prep_tab){ $('#myTab a[href="#'+top_tab+'"]').tab('show'); - if (top_tab == "raw_data_tab"){ + if (top_tab == "prep_template_tab"){ if (sub_tab !== undefined){ - $('#raw_data_nav_tabs a[href="#raw_data_info_'+sub_tab+'"]').tab('show') + $('#prep_template_nav_tabs a[href="#prep_template_info_'+sub_tab+'"]').tab('show') } } else if (top_tab == "preprocessed_data_tab"){ @@ -591,7 +587,7 @@

{{study_alias}}

Study information {% if show_data_tabs %} -
  • Raw data
    • +
  • Prep templates
  • Preprocessed data
  • Processed data
    • {% end %} @@ -603,7 +599,7 @@

    {{study_alias}}

    {% module StudyInformationTab(study) %} {% if show_data_tabs %} - {% module RawDataTab(study, full_access) %} + {% module PrepTemplateTab(study, full_access) %} {% module PreprocessedDataTab(study, full_access) %} diff --git a/qiita_pet/templates/study_description_templates/prep_template_tab.html b/qiita_pet/templates/study_description_templates/prep_template_tab.html new file mode 100644 index 000000000..b1a95ed7e --- /dev/null +++ b/qiita_pet/templates/study_description_templates/prep_template_tab.html @@ -0,0 +1,83 @@ +
    + + + + +
    + +
    + To add a prep template you need to: +
      + +
    1. + Select your prep template file (only files with the "txt" and "tsv" file extensions will be displayed here): + +
      2. + +
    2. + Select the prep template data type + +
      3. + +
    3. + Select an investigation type (optional but required for EBI submission): + + + + + + + + + + +
      +
      4. +
    +
    + Add prep template +
    + + + {% for _, _, pt, _ in available_prep_templates %} + {{pt.id}} + {% end %} + +
    + +
    diff --git a/qiita_pet/templates/study_description_templates/raw_data_tab.html b/qiita_pet/templates/study_description_templates/raw_data_tab.html deleted file mode 100644 index e8c6e3b3e..000000000 --- a/qiita_pet/templates/study_description_templates/raw_data_tab.html +++ /dev/null @@ -1,50 +0,0 @@ -
    - - - - -
    - -
    - What file type is your raw data? - - {% if other_studies_rd %} -

    - You could also select raw data from other studies that you have access to: -
    - - {% end %} -

    - Add raw data -
    - - - {% for _, _, rd, _ in available_raw_data %} - {% module RawDataEditorTab(study, rd, full_access) %} - {% end %} - -
    - -
    diff --git a/qiita_pet/uimodules/__init__.py b/qiita_pet/uimodules/__init__.py index 88f893074..91db58631 100644 --- a/qiita_pet/uimodules/__init__.py +++ b/qiita_pet/uimodules/__init__.py @@ -7,12 +7,12 @@ # ----------------------------------------------------------------------------- from .study_information_tab import StudyInformationTab -from .raw_data_tab import (RawDataTab, RawDataEditorTab, PrepTemplatePanel, - EditInvestigationType) +from .prep_template_tab import (PrepTemplateTab, RawDataEditorTab, + PrepTemplatePanel, EditInvestigationType) from .preprocessed_data_tab import PreprocessedDataTab, PreprocessedDataInfoTab from .processed_data_tab import ProcessedDataTab, ProcessedDataInfoTab -__all__ = ['StudyInformationTab', 'RawDataTab', 'RawDataEditorTab', +__all__ = ['StudyInformationTab', 'PrepTemplateTab', 'RawDataEditorTab', 'PrepTemplatePanel', 'EditInvestigationType', 'PreprocessedDataTab', 'PreprocessedDataInfoTab', 'ProcessedDataTab', 'ProcessedDataInfoTab'] diff --git a/qiita_pet/uimodules/raw_data_tab.py b/qiita_pet/uimodules/prep_template_tab.py similarity index 85% rename from qiita_pet/uimodules/raw_data_tab.py rename to qiita_pet/uimodules/prep_template_tab.py index b58882eb2..99e4fde34 100644 --- a/qiita_pet/uimodules/raw_data_tab.py +++ b/qiita_pet/uimodules/prep_template_tab.py @@ -41,24 +41,49 @@ def get_raw_data(rdis): return [RawData(rdi) for rdi in rdis] -class RawDataTab(BaseUIModule): +def get_prep_templates(pt_ids): + """Get all the prep template objects from a list of ids + + Parameters + ---------- + pt_ids : list of int + The prep template ids + + Returns + ------- + list of PrepTemplate + """ + return [PrepTemplate(pt_id) for pt_id in pt_ids] + + +class PrepTemplateTab(BaseUIModule): def render(self, study, full_access): user = self.current_user + files = [f for _, f in get_files_from_uploads_folders(str(study.id))] + data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) + prep_templates_info = [ + (pt.id, pt.data_type(), pt, STATUS_STYLER[pt.status]) + for pt in get_prep_templates(study.prep_templates()) + if full_access or pt.status() == 'public'] + # Get all the ENA terms for the investigation type + ontology = Ontology(convert_to_id('ENA', 'ontology')) + # make "Other" show at the bottom of the drop down menu + ena_terms = [] + for v in sorted(ontology.terms): + if v != 'Other': + ena_terms.append('' % (v, v)) + ena_terms.append('') - filetypes = sorted(viewitems(get_filetypes()), key=itemgetter(1)) - other_studies_rd = sorted(viewitems( - get_raw_data_from_other_studies(user, study))) - - raw_data_info = [ - (rd.id, rd.filetype, rd, STATUS_STYLER[rd.status(study)]) - for rd in get_raw_data(study.raw_data()) - if full_access or rd.status(study) == 'public'] + # New Type is for users to add a new user-defined investigation type + user_defined_terms = ontology.user_defined_terms + ['New Type'] return self.render_string( - "study_description_templates/raw_data_tab.html", - filetypes=filetypes, - other_studies_rd=other_studies_rd, - available_raw_data=raw_data_info, + "study_description_templates/prep_template_tab.html", + files=files, + data_types=data_types, + available_prep_templates=prep_templates_info, + ena_terms=ena_terms, + user_defined_terms=user_defined_terms, study=study, full_access=full_access) From d51b2998e3fe7015aea2982befd836e078b1a27b Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Fri, 22 May 2015 17:56:29 -0700 Subject: [PATCH 02/19] Hooking all related with raw data creation --- qiita_pet/handlers/compute.py | 51 +++++++++- qiita_pet/templates/study_description.html | 85 ++++++++++++++++- .../prep_template_info_tab.html | 94 +++++++++++++++++++ .../prep_template_tab.html | 2 +- qiita_pet/uimodules/__init__.py | 8 +- qiita_pet/uimodules/prep_template_tab.py | 71 +++++++++++++- qiita_pet/webserver.py | 3 +- qiita_ware/dispatchable.py | 8 ++ 8 files changed, 312 insertions(+), 10 deletions(-) create mode 100644 qiita_pet/templates/study_description_templates/prep_template_info_tab.html diff --git a/qiita_pet/handlers/compute.py b/qiita_pet/handlers/compute.py index 1cb16c29c..e48f5f41a 100644 --- a/qiita_pet/handlers/compute.py +++ b/qiita_pet/handlers/compute.py @@ -7,11 +7,13 @@ from .util import check_access from qiita_ware.context import submit -from qiita_ware.dispatchable import add_files_to_raw_data, unlink_all_files +from qiita_ware.dispatchable import (add_files_to_raw_data, unlink_all_files, + create_raw_data) from qiita_db.study import Study from qiita_db.exceptions import QiitaDBUnknownIDError from qiita_db.util import get_mountpoint +from qiita_db.metadata_template import PrepTemplate from os.path import join, exists @@ -28,6 +30,53 @@ def get(self, job_id): self.redirect('/') +class CreateRawData(BaseHandler): + @authenticated + def post(self): + pt_id = self.get_argument('prep_template_id') + raw_data_filetype = self.get_argument('filetype') + barcodes_str = self.get_argument('barcodes') + forward_reads_str = self.get_argument('forward') + sff_str = self.get_argument('sff') + fasta_str = self.get_argument('fasta') + qual_str = self.get_argument('qual') + reverse_reads_str = self.get_argument('reverse') + + pt = PrepTemplate(pt_id) + study_id = pt.study_id + + def _split(x): + return x.split(',') if x else [] + filepaths, fps = [], [] + fps.append((_split(barcodes_str), 'raw_barcodes')) + fps.append((_split(fasta_str), 'raw_fasta')) + fps.append((_split(qual_str), 'raw_qual')) + fps.append((_split(forward_reads_str), 'raw_forward_seqs')) + fps.append((_split(reverse_reads_str), 'raw_reverse_seqs')) + fps.append((_split(sff_str), 'raw_sff')) + + for _, f in get_mountpoint("uploads", retrieve_all=True): + f = join(f, str(study_id)) + for fp_set, filetype in fps: + for t in fp_set: + ft = join(f, t) + if exists(ft): + filepaths.append((ft, filetype)) + + print raw_data_filetype + print pt + print filepaths + + job_id = submit(self.current_user.id, create_raw_data, + raw_data_filetype, pt, filepaths) + + self.render('compute_wait.html', + job_id=job_id, title='Adding raw data', + completion_redirect=( + '/study/description/%s?top_tab=raw_data_tab&sub_tab=%s' + % (study_id, pt_id))) + + class AddFilesToRawData(BaseHandler): @authenticated def post(self): diff --git a/qiita_pet/templates/study_description.html b/qiita_pet/templates/study_description.html index 3cd9ee4f2..7d8570bae 100644 --- a/qiita_pet/templates/study_description.html +++ b/qiita_pet/templates/study_description.html @@ -111,7 +111,89 @@ form.submit(); } -function create_raw_data() { +function create_raw_data(prep_id, filetype){ + var barcodes = []; + var forward = []; + var reverse = []; + var sff = []; + var fasta = []; + var qual = []; + + var all_files = document.getElementsByName("upload_file_" + prep_id + "_" + filetype); + + for (var i = 0; i < all_files.length; i++){ + ele = all_files[i] + switch (ele.options[ele.selectedIndex].value) { + case "barcodes": + barcodes.push(ele.id); + break; + case "forward seqs": + forward.push(ele.id); + break; + case "reverse seqs": + reverse.push(ele.id); + break; + case "sff": + sff.push(ele.id); + break; + case "fasta": + fasta.push(ele.id); + break; + case "qual": + qual.push(ele.id); + break; + } + } + if (sff.length === 0 && barcodes.length === 0 && forward.length === 0 && fasta.length === 0) { + bootstrapAlert("You need to select at least: one barcode and one forward file, one sff file or one fasta file."); + } else if (sff.length === 0 && barcodes.length != forward.length) { + bootstrapAlert("You must select the same number of barcode and forward seq files."); + } else if (sff.length === 0 && (reverse.length !== 0 && reverse.length != forward.length)) { + bootstrapAlert("If you select reverse seqs, they should have the same number than barcodes and forward files."); + } else if (sff.length === 0 && (qual.length !== 0 && fasta.length != qual.length)) { + bootstrapAlert("If you select qual, you must have the same number of fasta and qual files."); + } else { + var form = $("") + .attr("action", "/study/create_raw_data") + .attr("method", "post") + .append($("") + .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "filetype") + .attr("value", filetype)) + .append($("") + .attr("type", "hidden") + .attr("name", "barcodes") + .attr("value", barcodes.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "forward") + .attr("value", forward.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "sff") + .attr("value", sff.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "fasta") + .attr("value", fasta.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "qual") + .attr("value", qual.join())) + .append($("") + .attr("type", "hidden") + .attr("name", "reverse") + .attr("value", reverse.join())); + $("body").append(form); + form.submit(); + } +} + +function create_raw_data_old() { if ($("#previous_raw_data").val() === null && $("#filetype").val() === "") { bootstrapAlert("You need to select either a new file type or a raw data used in a previous study."); } else if ((typeof $("#previous_raw_data").val() != 'undefined' && $("#previous_raw_data").val() !== null) && $("#filetype").val() !== "") { @@ -551,6 +633,7 @@ , "{{prep_tab}}" {% end %} {% end %}) + $('.filesgroup').hide(); }); diff --git a/qiita_pet/templates/study_description_templates/prep_template_info_tab.html b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html new file mode 100644 index 000000000..94efc937d --- /dev/null +++ b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html @@ -0,0 +1,94 @@ +
    + + + + + + + {% if raw_data %} + + + + {% else %} + + + + + + + {% end %} +
    + + Show prep template summary + +
    {% raw current_template_fp %}
    {% raw current_qiime_fp %}
    Raw data: {{raw_data}}
    + +
    +
    +
    +
    + +
    +
    + What file type is your raw data? + +
    + {% for _, value, filepath_types in filetypes %} +
    + + + + + + + + {% for f in files %} + + + + + + {% end %} + +
    Choose the files to link to the raw data
    File   File type
    {{f}}  + +
    Add
    +
    + {% end %} +
    +
    +
    +
    + +
    +
    + Use existing one! +
    +
    +
    +
    +
    +
    +
    diff --git a/qiita_pet/templates/study_description_templates/prep_template_tab.html b/qiita_pet/templates/study_description_templates/prep_template_tab.html index b1a95ed7e..9ab74e72b 100644 --- a/qiita_pet/templates/study_description_templates/prep_template_tab.html +++ b/qiita_pet/templates/study_description_templates/prep_template_tab.html @@ -75,7 +75,7 @@ {% for _, _, pt, _ in available_prep_templates %} - {{pt.id}} + {% module PrepTemplateInfoTab(study, pt, full_access) %} {% end %} diff --git a/qiita_pet/uimodules/__init__.py b/qiita_pet/uimodules/__init__.py index 91db58631..8a007657d 100644 --- a/qiita_pet/uimodules/__init__.py +++ b/qiita_pet/uimodules/__init__.py @@ -7,12 +7,12 @@ # ----------------------------------------------------------------------------- from .study_information_tab import StudyInformationTab -from .prep_template_tab import (PrepTemplateTab, RawDataEditorTab, - PrepTemplatePanel, EditInvestigationType) +from .prep_template_tab import (PrepTemplateTab, PrepTemplateInfoTab, + EditInvestigationType) from .preprocessed_data_tab import PreprocessedDataTab, PreprocessedDataInfoTab from .processed_data_tab import ProcessedDataTab, ProcessedDataInfoTab -__all__ = ['StudyInformationTab', 'PrepTemplateTab', 'RawDataEditorTab', - 'PrepTemplatePanel', 'EditInvestigationType', 'PreprocessedDataTab', +__all__ = ['StudyInformationTab', 'PrepTemplateTab', 'PrepTemplateInfoTab', + 'EditInvestigationType', 'PreprocessedDataTab', 'PreprocessedDataInfoTab', 'ProcessedDataTab', 'ProcessedDataInfoTab'] diff --git a/qiita_pet/uimodules/prep_template_tab.py b/qiita_pet/uimodules/prep_template_tab.py index 99e4fde34..225cb4abe 100644 --- a/qiita_pet/uimodules/prep_template_tab.py +++ b/qiita_pet/uimodules/prep_template_tab.py @@ -8,6 +8,7 @@ from operator import itemgetter from os.path import basename +from collections import defaultdict from future.utils import viewitems @@ -20,6 +21,7 @@ from qiita_db.parameters import (Preprocessed454Params, PreprocessedIlluminaParams) from qiita_pet.util import STATUS_STYLER +from qiita_pet.handlers.util import download_link_or_path from .base_uimodule import BaseUIModule @@ -53,12 +55,11 @@ def get_prep_templates(pt_ids): ------- list of PrepTemplate """ - return [PrepTemplate(pt_id) for pt_id in pt_ids] + return [PrepTemplate(pt_id) for pt_id in sorted(pt_ids)] class PrepTemplateTab(BaseUIModule): def render(self, study, full_access): - user = self.current_user files = [f for _, f in get_files_from_uploads_folders(str(study.id))] data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) prep_templates_info = [ @@ -88,6 +89,72 @@ def render(self, study, full_access): full_access=full_access) +class PrepTemplateInfoTab(BaseUIModule): + def render(self, study, prep_template, full_access): + is_local_request = self._is_local() + + template_fps = [] + qiime_fps = [] + # Unfortunately, both the prep template and the qiime mapping files + # have the sample type. The way to differentiate them is if we have + # the substring 'qiime' in the basename + for id_, fp in prep_template.get_filepaths(): + if 'qiime' in basename(fp): + qiime_fps.append( + download_link_or_path( + is_local_request, fp, id_, 'Qiime mapping')) + else: + template_fps.append( + download_link_or_path( + is_local_request, fp, id_, 'Prep template')) + + # Since get_filepaths returns the paths sorted from newest to oldest, + # the first in both list is the latest one + current_template_fp = template_fps[0] + current_qiime_fp = qiime_fps[0] + + if len(template_fps) > 1: + show_old_templates = True + old_templates = template_fps[1:] + else: + show_old_templates = False + old_templates = None + + if len(qiime_fps) > 1: + show_old_qiime_fps = True + old_qiime_fps = qiime_fps[1:] + else: + show_old_qiime_fps = False + old_qiime_fps = None + + filepath_types = [k.split('_', 1)[1].replace('_', ' ') + for k in get_filepath_types() + if k.startswith('raw_')] + fp_type_by_ft = defaultdict( + lambda: filepath_types, SFF=['sff'], FASTA=['fasta', 'qual'], + FASTQ=['barcodes', 'forward seqs', 'reverse seqs']) + + filetypes = sorted( + ((ft, ft_id, fp_type_by_ft[ft]) + for ft, ft_id in viewitems(get_filetypes())), + key=itemgetter(1)) + files = [f for _, f in get_files_from_uploads_folders(str(study.id))] + + return self.render_string( + "study_description_templates/prep_template_info_tab.html", + pt_id=prep_template.id, + study_id=study.id, + raw_data=prep_template.raw_data, + current_template_fp=current_template_fp, + current_qiime_fp=current_qiime_fp, + show_old_templates=show_old_templates, + old_templates=old_templates, + show_old_qiime_fps=show_old_qiime_fps, + old_qiime_fps=old_qiime_fps, + filetypes=filetypes, + files=files) + + class RawDataEditorTab(BaseUIModule): def render(self, study, raw_data, full_access): user = self.current_user diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 9097fe1d5..324c31b86 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -28,7 +28,7 @@ from qiita_pet.handlers.logger_handlers import LogEntryViewerHandler from qiita_pet.handlers.upload import UploadFileHandler, StudyUploadFileHandler from qiita_pet.handlers.compute import ( - ComputeCompleteHandler, AddFilesToRawData, UnlinkAllFiles) + ComputeCompleteHandler, AddFilesToRawData, UnlinkAllFiles, CreateRawData) from qiita_pet.handlers.preprocessing_handlers import PreprocessHandler from qiita_pet.handlers.processing_handlers import ProcessHandler from qiita_pet.handlers.stats import StatsHandler @@ -81,6 +81,7 @@ def __init__(self): (r"/study/list/socket/", SelectSamplesHandler), (r"/study/search/(.*)", SearchStudiesAJAX), (r"/study/add_files_to_raw_data", AddFilesToRawData), + (r"/study/create_raw_data", CreateRawData), (r"/study/unlink_all_files", UnlinkAllFiles), (r"/study/preprocess", PreprocessHandler), (r"/study/process", ProcessHandler), diff --git a/qiita_ware/dispatchable.py b/qiita_ware/dispatchable.py index 7ff497bee..0c5883541 100644 --- a/qiita_ware/dispatchable.py +++ b/qiita_ware/dispatchable.py @@ -72,6 +72,14 @@ def run_analysis(analysis_id, commands, comm_opts=None, return ar(analysis, commands, comm_opts, rarefaction_depth) +def create_raw_data(filetype, prep_template, filepaths): + """Creates a new raw data + + Needs to be dispachable because it moves large files + """ + RawData.create(filetype, [prep_template], filepaths) + + def add_files_to_raw_data(raw_data_id, filepaths): """Add files to raw data From 2720331572f70b4c3621b20a150e889a4bec959a Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Fri, 22 May 2015 18:56:52 -0700 Subject: [PATCH 03/19] Adding all bits for raw data deletion --- .../study_handlers/description_handlers.py | 13 +- qiita_pet/templates/study_description.html | 8 +- .../prep_template_info_tab.html | 12 +- .../raw_data_info.html | 25 +++ qiita_pet/uimodules/__init__.py | 4 +- qiita_pet/uimodules/prep_template_tab.py | 172 ++---------------- 6 files changed, 59 insertions(+), 175 deletions(-) create mode 100644 qiita_pet/templates/study_description_templates/raw_data_info.html diff --git a/qiita_pet/handlers/study_handlers/description_handlers.py b/qiita_pet/handlers/study_handlers/description_handlers.py index c49b86c1f..b30796fd7 100644 --- a/qiita_pet/handlers/study_handlers/description_handlers.py +++ b/qiita_pet/handlers/study_handlers/description_handlers.py @@ -711,21 +711,18 @@ def delete_raw_data(self, study, user, callback): is done """ raw_data_id = int(self.get_argument('raw_data_id')) + prep_template_id = int(self.get_argument('prep_template_id')) try: - RawData.delete(raw_data_id, study.id) - msg = ("Raw data %d has been deleted from study: " - "%s" % (raw_data_id, study.title)) + RawData.delete(raw_data_id, prep_template_id) + msg = ("Raw data %d has been deleted from prep_template %d" + % (raw_data_id, prep_template_id)) msg_level = "success" - tab = 'study_information_tab' - tab_id = None except Exception as e: msg = "Couldn't remove %d raw data: %s" % (raw_data_id, str(e)) msg_level = "danger" - tab = 'raw_data_tab' - tab_id = raw_data_id - callback((msg, msg_level, tab, tab_id, None)) + callback((msg, msg_level, "prep_template_tab", prep_template_id, None)) def delete_prep_template(self, study, user, callback): """Delete the selected prep template diff --git a/qiita_pet/templates/study_description.html b/qiita_pet/templates/study_description.html index 7d8570bae..173133b51 100644 --- a/qiita_pet/templates/study_description.html +++ b/qiita_pet/templates/study_description.html @@ -311,7 +311,7 @@ } } -function delete_raw_data(raw_data_filetype, raw_data_id) { +function delete_raw_data(raw_data_filetype, raw_data_id, prep_template_id) { if (confirm('Are you sure you want to delete raw data: ' + raw_data_filetype + ' (ID: ' + raw_data_id + ')?')) { var form = $("") .attr("action", window.location.href) @@ -322,6 +322,10 @@ .attr("value", raw_data_id)) .append($("") .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_template_id)) + .append($("") + .attr("type", "hidden") .attr("name", "action") .attr("value", "delete_raw_data")); $("body").append(form); @@ -580,7 +584,7 @@ } function unlink_all_files(raw_data_id) { - if (confirm("Are you sure you want to unlink all files? They will be removed from the system")){ + if (confirm("Are you sure you want to unlink all files?")){ var form = $("") .attr("action", "/study/unlink_all_files") .attr("method", "post") diff --git a/qiita_pet/templates/study_description_templates/prep_template_info_tab.html b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html index 94efc937d..543b51a94 100644 --- a/qiita_pet/templates/study_description_templates/prep_template_info_tab.html +++ b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html @@ -1,26 +1,26 @@
    - - - + + {% if raw_data %} - + {% else %} - -
    + Show prep template summary
    {% raw current_template_fp %}
    {% raw current_qiime_fp %}
    {% raw current_template_fp %}
    {% raw current_qiime_fp %}
    Raw data: {{raw_data}}{% module RawDataInfoDiv(raw_data, prep_template, study) %}
    +
    +
    diff --git a/qiita_pet/templates/study_description_templates/raw_data_info.html b/qiita_pet/templates/study_description_templates/raw_data_info.html new file mode 100644 index 000000000..ed514919c --- /dev/null +++ b/qiita_pet/templates/study_description_templates/raw_data_info.html @@ -0,0 +1,25 @@ +
    +
    +
    + +
    +
    +

    Files linked to this raw data:


    + {% if show_unlink_btn %} + Unlink all files
    + {% end %} + {% for fname, ftype in raw_data_files %} + {{fname}}: {{ftype}}
    + {% end %} +
    +
    +
    +
    +
    \ No newline at end of file diff --git a/qiita_pet/uimodules/__init__.py b/qiita_pet/uimodules/__init__.py index 8a007657d..43fcf9724 100644 --- a/qiita_pet/uimodules/__init__.py +++ b/qiita_pet/uimodules/__init__.py @@ -8,11 +8,11 @@ from .study_information_tab import StudyInformationTab from .prep_template_tab import (PrepTemplateTab, PrepTemplateInfoTab, - EditInvestigationType) + RawDataInfoDiv, EditInvestigationType) from .preprocessed_data_tab import PreprocessedDataTab, PreprocessedDataInfoTab from .processed_data_tab import ProcessedDataTab, ProcessedDataInfoTab __all__ = ['StudyInformationTab', 'PrepTemplateTab', 'PrepTemplateInfoTab', - 'EditInvestigationType', 'PreprocessedDataTab', + 'EditInvestigationType', 'RawDataInfoDiv', 'PreprocessedDataTab', 'PreprocessedDataInfoTab', 'ProcessedDataTab', 'ProcessedDataInfoTab'] diff --git a/qiita_pet/uimodules/prep_template_tab.py b/qiita_pet/uimodules/prep_template_tab.py index 225cb4abe..3b0d592a6 100644 --- a/qiita_pet/uimodules/prep_template_tab.py +++ b/qiita_pet/uimodules/prep_template_tab.py @@ -152,167 +152,25 @@ def render(self, study, prep_template, full_access): show_old_qiime_fps=show_old_qiime_fps, old_qiime_fps=old_qiime_fps, filetypes=filetypes, - files=files) - - -class RawDataEditorTab(BaseUIModule): - def render(self, study, raw_data, full_access): - user = self.current_user - study_status = study.status - user_level = user.level - raw_data_id = raw_data.id - files = [f for _, f in get_files_from_uploads_folders(str(study.id))] - - # Get the available prep template data types - data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) - - # Get all the ENA terms for the investigation type - ontology = Ontology(convert_to_id('ENA', 'ontology')) - # make "Other" show at the bottom of the drop down menu - ena_terms = [] - for v in sorted(ontology.terms): - if v != 'Other': - ena_terms.append('' % (v, v)) - ena_terms.append('') - - # New Type is for users to add a new user-defined investigation type - user_defined_terms = ontology.user_defined_terms + ['New Type'] - - # Get all the information about the prep templates - available_prep_templates = [] - for p in sorted(raw_data.prep_templates): - if PrepTemplate.exists(p): - pt = PrepTemplate(p) - # if the prep template doesn't belong to this study, skip - if (study.id == pt.study_id and - (full_access or pt.status == 'public')): - available_prep_templates.append(pt) - - # getting filepath_types - if raw_data.filetype == 'SFF': - fts = ['sff'] - elif raw_data.filetype == 'FASTA': - fts = ['fasta', 'qual'] - elif raw_data.filetype == 'FASTQ': - fts = ['barcodes', 'forward seqs', 'reverse seqs'] - else: - fts = [k.split('_', 1)[1].replace('_', ' ') - for k in get_filepath_types() if k.startswith('raw_')] - - # The raw data can be edited (e.i. adding prep templates and files) - # only if the study is sandboxed or the current user is an admin - is_editable = study_status == 'sandbox' or user_level == 'admin' - - # Get the files linked with the raw_data - raw_data_files = raw_data.get_filepaths() - - # Get the status of the data linking - raw_data_link_status = raw_data.link_filepaths_status - - # By default don't show the unlink button - show_unlink_btn = False - # By default disable the the link file button - disable_link_btn = True - # Define the message for the link status - if raw_data_link_status == 'linking': - link_msg = "Linking files..." - elif raw_data_link_status == 'unlinking': - link_msg = "Unlinking files..." - else: - # The link button is only disable if raw data link status is - # linking or unlinking, so we can enable it here - disable_link_btn = False - # The unlink button is only shown if the study is editable, the raw - # data linking status is not in linking or unlinking, and there are - # files attached to the raw data. At this point, we are sure that - # the raw data linking status is not in linking or unlinking so we - # still need to check if it is editable or there are files attached - show_unlink_btn = is_editable and raw_data_files - if raw_data_link_status.startswith('failed'): - link_msg = "Error (un)linking files: %s" % raw_data_link_status - else: - link_msg = "" - - # Get the raw_data filetype - raw_data_filetype = raw_data.filetype - - return self.render_string( - "study_description_templates/raw_data_editor_tab.html", - study_id=study.id, - study_status=study_status, - user_level=user_level, - raw_data_id=raw_data_id, files=files, - data_types=data_types, - ena_terms=ena_terms, - user_defined_terms=user_defined_terms, - available_prep_templates=available_prep_templates, - filepath_types=fts, - is_editable=is_editable, - show_unlink_btn=show_unlink_btn, - link_msg=link_msg, - raw_data_files=raw_data_files, - raw_data_filetype=raw_data_filetype, - disable_link_btn=disable_link_btn) - - -class PrepTemplatePanel(BaseUIModule): - def render(self, prep, study_id, is_editable, ena_terms, - study_status, user_defined_terms): - # Check if the request came from a local source - is_local_request = self._is_local() - - prep_id = prep.id - status_class1, status_class2, status_color = STATUS_STYLER[prep.status] - data_type = prep.data_type() - raw_data = RawData(prep.raw_data) - filepaths = prep.get_filepaths() - investigation_type = prep.investigation_type - preprocessed_data = prep.preprocessed_data - preprocessing_status = prep.preprocessing_status - - if raw_data.filetype in ('SFF', 'FASTA'): - param_iter = Preprocessed454Params.iter() - elif raw_data.filetype == 'FASTQ': - param_iter = PreprocessedIlluminaParams.iter() - else: - raise ValueError("Don't know what to do but this exception will " - "never actually get shown anywhere because why " - "would you want to see tracebacks?") - - preprocess_options = [] - for param in param_iter: - text = ("%s: %s" % (k, v) - for k, v in viewitems(param.values)) - preprocess_options.append((param.id, - param.name, - '
    '.join(text))) + prep_template=prep_template, + study=study) - # Unfortunately, both the prep template and the qiime mapping files - # have the sample type. The way to differentiate them is if we have - # the substring 'qiime' in the basename - _fp_type = (lambda fp: "Qiime mapping" - if 'qiime' in basename(fp) else "Prep template") - filepaths = [(id_, fp, _fp_type(fp)) for id_, fp in filepaths] +class RawDataInfoDiv(BaseUIModule): + def render(self, raw_data_id, prep_template, study): + rd = RawData(raw_data_id) + raw_data_files = [(basename(fp), fp_type[4:]) + for _, fp, fp_type in rd.get_filepaths()] + show_unlink_btn = (rd.status(study) == 'sandbox' and + raw_data_files) return self.render_string( - "study_description_templates/prep_template_panel.html", - prep_id=prep_id, - status_class1=status_class1, - status_class2=status_class2, - status_color=status_color, - data_type=data_type, - filepaths=filepaths, - investigation_type=investigation_type, - preprocessed_data=preprocessed_data, - preprocessing_status=preprocessing_status, - study_id=study_id, - is_local_request=is_local_request, - is_editable=is_editable, - ena_terms=ena_terms, - study_status=study_status, - user_defined_terms=user_defined_terms, - preprocess_options=preprocess_options) + "study_description_templates/raw_data_info.html", + rd_id=raw_data_id, + rd_filetype=rd.filetype, + raw_data_files=raw_data_files, + prep_template_id=prep_template.id, + show_unlink_btn=show_unlink_btn) class EditInvestigationType(BaseUIModule): From c85e180d043c32bf0f68fb8c7dade4733949a985 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Sat, 23 May 2015 09:04:25 -0700 Subject: [PATCH 04/19] Adding bits for adding an existing raw data to a prep template --- .../study_handlers/description_handlers.py | 32 +++++++++++++++++++ qiita_pet/templates/study_description.html | 25 +++++++++++++++ .../prep_template_info_tab.html | 14 +++++++- .../raw_data_info.html | 8 ++--- qiita_pet/uimodules/prep_template_tab.py | 9 ++++-- 5 files changed, 80 insertions(+), 8 deletions(-) diff --git a/qiita_pet/handlers/study_handlers/description_handlers.py b/qiita_pet/handlers/study_handlers/description_handlers.py index b30796fd7..883b3d246 100644 --- a/qiita_pet/handlers/study_handlers/description_handlers.py +++ b/qiita_pet/handlers/study_handlers/description_handlers.py @@ -311,6 +311,37 @@ def add_to_sample_template(self, study, user, callback): callback((msg, msg_level, None, None, None)) + def add_raw_data(self, study, user, callback): + """Adds an existing raw data to the study + + Parameters + ---------- + study : Study + The current study object + user : User + The current user object + callback : function + The callback function to call with the results once the processing + is done + """ + msg = "Raw data successfully added" + msg_level = "success" + + # Get the arguments to add the raw data + pt_id = self.get_argument('prep_template_id') + raw_data_id = self.get_argument('raw_data_id') + + prep_template = PrepTemplate(pt_id) + raw_data = RawData(raw_data_id) + + try: + prep_template.raw_data = raw_data + except QiitaDBError as e: + msg = html_error_message % ("adding the raw data", + str(raw_data_id), str(e)) + + callback((msg, msg_level, 'prep_template_tab', pt_id, None)) + def create_raw_data(self, study, user, callback): """Adds a (new) raw data to the study @@ -829,6 +860,7 @@ def post(self, study_id): update_sample_template=self.update_sample_template, extend_sample_template=self.add_to_sample_template, create_raw_data=self.create_raw_data, + add_raw_data=self.add_raw_data, add_prep_template=self.add_prep_template, make_public=self.make_public, approve_study=self.approve_study, diff --git a/qiita_pet/templates/study_description.html b/qiita_pet/templates/study_description.html index 173133b51..17a808b61 100644 --- a/qiita_pet/templates/study_description.html +++ b/qiita_pet/templates/study_description.html @@ -193,6 +193,31 @@ } } +function add_raw_data(prep_id) { + raw_data_id = $("#previous-raw-data-" + prep_id).val() + if (raw_data_id === null){ + bootstrapAlert("You need to select a raw data used in a previous study"); + } else { + var form = $("") + .attr("action", window.location.href) + .attr("method", "post") + .append($("") + .attr("type", "hidden") + .attr("name", "raw_data_id") + .attr("value", raw_data_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "prep_template_id") + .attr("value", prep_id)) + .append($("") + .attr("type", "hidden") + .attr("name", "action") + .attr("value", "add_raw_data")); + $("body").append(form); + form.submit(); + } +} + function create_raw_data_old() { if ($("#previous_raw_data").val() === null && $("#filetype").val() === "") { bootstrapAlert("You need to select either a new file type or a raw data used in a previous study."); diff --git a/qiita_pet/templates/study_description_templates/prep_template_info_tab.html b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html index 543b51a94..c0415a470 100644 --- a/qiita_pet/templates/study_description_templates/prep_template_info_tab.html +++ b/qiita_pet/templates/study_description_templates/prep_template_info_tab.html @@ -81,7 +81,19 @@

    - Use existing one! + {% if other_studies_rd %} + Choose raw data from studies that you have access to: +
    + +
    + Add + {% else %} + There is no existent raw data available. + {% end %}
    diff --git a/qiita_pet/templates/study_description_templates/raw_data_info.html b/qiita_pet/templates/study_description_templates/raw_data_info.html index ed514919c..1b9e38c58 100644 --- a/qiita_pet/templates/study_description_templates/raw_data_info.html +++ b/qiita_pet/templates/study_description_templates/raw_data_info.html @@ -1,15 +1,15 @@
    -
    +
    -