From 2215406447eb9545a698b6abfd40590a962b1034 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 6 Oct 2015 08:44:47 -0600 Subject: [PATCH 01/16] fix #1479 --- qiita_ware/ebi.py | 33 ++++++++++++++--------- qiita_ware/test/test_ebi.py | 53 +++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 0891a545f..13cce9319 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -16,7 +16,7 @@ from qiita_ware.util import open_file from qiita_db.logger import LogEntry from qiita_db.ontology import Ontology -from qiita_db.util import convert_to_id +from qiita_db.util import convert_to_id, get_mountpoint, insert_filepaths from qiita_db.study import Study from qiita_db.data import PreprocessedData from qiita_db.metadata_template import PrepTemplate, SampleTemplate @@ -157,10 +157,10 @@ def __init__(self, preprocessed_data_id, action): "term is neither one of the official terms nor " "one of the user-defined terms in the ENA " "ontology." % it) - - self.ebi_dir = join(qiita_config.working_dir, - 'ebi_submission_%d' % preprocessed_data_id) - get_output_fp = partial(join, self.ebi_dir, 'xml_dir') + _, base_fp = get_mountpoint("preprocessed_data")[0] + self.ebi_dir = '%d_ebi_submission' % preprocessed_data_id + self.full_ebi_dir = join(base_fp, self.ebi_dir) + get_output_fp = partial(join, self.full_ebi_dir, 'xml_dir') self.xml_dir = get_output_fp() self.study_xml_fp = get_output_fp('study.xml') self.sample_xml_fp = get_output_fp('sample.xml') @@ -186,7 +186,7 @@ def __init__(self, preprocessed_data_id, action): self.samples = {} self.samples_prep = {} self.sample_demux_fps = {} - get_output_fp = partial(join, self.ebi_dir) + get_output_fp = partial(join, self.full_ebi_dir) nvp = [] nvim = [] for k, v in viewitems(self.sample_template): @@ -535,7 +535,7 @@ def generate_run_xml(self): data_block = ET.SubElement(run, 'DATA_BLOCK') files = ET.SubElement(data_block, 'FILES') ET.SubElement(files, 'FILE', { - 'filename': file_path, + 'filename': join(self.ebi_dir, basename(file_path)), 'filetype': file_type, 'quality_scoring_system': 'phred', 'checksum_method': 'MD5', @@ -787,10 +787,10 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): Notes ----- - - As a performace feature, this method will check if self.ebi_dir + - As a performace feature, this method will check if self.full_ebi_dir already exists and, if it does, the script will assume that in a previous execution this step was performed correctly and will simply - read the file names from self.ebi_dir + read the file names from self.full_ebi_dir - When the object is created (init), samples, samples_prep and sample_demux_fps hold values for all available samples in the database. Here some of those values will be deleted (del's, within the loops) for @@ -800,8 +800,14 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): """ ppd = self.preprocessed_data - if not isdir(self.ebi_dir) or rewrite_fastq: - makedirs(self.ebi_dir) + dir_not_exists = not isdir(self.full_ebi_dir) + if dir_not_exists or rewrite_fastq: + if dir_not_exists: + makedirs(self.full_ebi_dir) + filetype_id = convert_to_id('directory', 'filepath_type') + insert_filepaths([(self.full_ebi_dir, filetype_id)], -1, + 'preprocessed_data', 'filepath', + move_files=False) demux = [path for _, path, ftype in ppd.get_filepaths() if ftype == 'preprocessed_demux'][0] @@ -829,8 +835,9 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): demux_samples = set() extension = '.fastq.gz' extension_len = len(extension) - for f in listdir(self.ebi_dir): - if isfile(join(self.ebi_dir, f)) and f.endswith(extension): + for f in listdir(self.full_ebi_dir): + fpath = join(self.full_ebi_dir, f) + if isfile(fpath) and f.endswith(extension): demux_samples.add(f[:-extension_len]) missing_samples = set(self.samples.keys()).difference( diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index a2450c220..395de01c3 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -30,6 +30,7 @@ from qiita_db.data import PreprocessedData from qiita_db.study import Study from qiita_db.metadata_template import PrepTemplate +from qiita_db.util import get_mountpoint from qiita_core.util import qiita_test_checker @@ -75,16 +76,14 @@ def test_init(self): self.assertItemsEqual(e.pmids, ['123456', '7891011']) self.assertEqual(e.action, action) - get_output_fp = partial(join, e.ebi_dir, 'xml_dir') - self.assertEqual(e.xml_dir, get_output_fp()) + get_output_fp = partial(join, e.full_ebi_dir, 'xml_dir') self.assertEqual(e.study_xml_fp, get_output_fp('study.xml')) self.assertEqual(e.sample_xml_fp, get_output_fp('sample.xml')) self.assertEqual(e.experiment_xml_fp, get_output_fp('experiment.xml')) self.assertEqual(e.run_xml_fp, get_output_fp('run.xml')) self.assertEqual(e.submission_xml_fp, get_output_fp('submission.xml')) - get_output_fp = partial(join, e.ebi_dir) - self.assertEqual(e.ebi_dir, get_output_fp()) + get_output_fp = partial(join, e.full_ebi_dir) for sample in e.sample_template: self.assertEqual(e.sample_template[sample], e.samples[sample]) self.assertEqual(e.prep_template[sample], e.samples_prep[sample]) @@ -377,7 +376,7 @@ def test_generate_run_xml(self): del(submission.samples_prep[k]) submission.generate_demultiplexed_fastq(mtime=1) - self.files_to_remove.append(submission.ebi_dir) + self.files_to_remove.append(submission.full_ebi_dir) obs = ET.tostring(submission.generate_run_xml()) exp = RUNXML % { @@ -400,7 +399,7 @@ def test_generate_demultiplexed_fastq(self): # be created ebi_submission = EBISubmission(ppd.id, 'ADD') obs_demux_samples = ebi_submission.generate_demultiplexed_fastq() - self.files_to_remove.append(ebi_submission.ebi_dir) + self.files_to_remove.append(ebi_submission.full_ebi_dir) self.assertItemsEqual(obs_demux_samples, exp_demux_samples) # testing that the samples/samples_prep and demux_samples are the same self.assertItemsEqual(obs_demux_samples, ebi_submission.samples.keys()) @@ -421,40 +420,42 @@ def test_generate_send_sequences_cmd(self): ppd = self.write_demux_files(PrepTemplate(1)) e = EBISubmission(ppd.id, 'ADD') e.generate_demultiplexed_fastq() - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) e.write_xml_file(e.generate_study_xml(), e.study_xml_fp) e.write_xml_file(e.generate_sample_xml(), e.sample_xml_fp) e.write_xml_file(e.generate_experiment_xml(), e.experiment_xml_fp) e.write_xml_file(e.generate_run_xml(), e.run_xml_fp) e.write_xml_file(e.generate_submission_xml(), e.submission_xml_fp) obs = e.generate_send_sequences_cmd() - exp = ['ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKB2.640194.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + _, base_fp = get_mountpoint("preprocessed_data")[0] + exp = ('ascp --ignore-host-key -L- -d -QT -k2 ' + '%(ebi_dir)s/1.SKB2.640194.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM4.640180.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + '%(ebi_dir)s/1.SKM4.640180.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKB3.640195.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + '%(ebi_dir)s/1.SKB3.640195.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKB6.640176.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + '%(ebi_dir)s/1.SKB6.640176.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKD6.640190.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + '%(ebi_dir)s/1.SKD6.640190.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM6.640187.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + '%(ebi_dir)s/1.SKM6.640187.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKD9.640182.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + '%(ebi_dir)s/1.SKD9.640182.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM8.640201.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', + '%(ebi_dir)s/1.SKM8.640201.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM2.640199.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/'] + '%(ebi_dir)s/1.SKM2.640199.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/' % { + 'ebi_dir': e.full_ebi_dir}).split('\n') self.assertEqual(obs, exp) def test_parse_EBI_reply(self): From c0c3caa2ca4185ac9ed67de948df88d40f4a15ed Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 6 Oct 2015 08:50:02 -0600 Subject: [PATCH 02/16] description is required for EBI --- .../support_files/doc/source/tutorials/prepare-templates.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst index e98f38e67..cc84b3ea1 100644 --- a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst +++ b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst @@ -52,8 +52,6 @@ the centralized `Qiita server `__: +===================================+======================================================================+=============================================================================================================================================================================================================================================+ | ``sample_type`` | free text | Description of the type of sample. | +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``description`` | free text | Description of the sample. | -+-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``physical_specimen_remaining`` | ``y`` or ``n`` | Is there still physical sample (e.g., soil, not DNA) available? | +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``dna_extracted`` | ``y`` or ``n`` | Has DNA already been extracted for this sample? | @@ -98,6 +96,8 @@ These are the columns required for successfully submit your data to EBI: +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``library_construction_protocol`` | free text | Brief description or reference to the protocol that was used for preparing this amplicon library starting from DNA, usually this includes what genomic region was targeted such as *16S*, *ITS*, *18S*, etc. | +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``description`` | free text | Description of the sample. | ++-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``experiment_design_description`` | free text | High-level description of the study (for example, *A longitudinal study of the gut microbiome of two human subjects*). | +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ From 42b73ed1ce67666c48eaf0b304911dcddb410731 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 6 Oct 2015 12:55:28 -0600 Subject: [PATCH 03/16] removing unnecessary EBI cfg vars --- qiita_core/configuration_manager.py | 7 ------- qiita_core/support_files/config_test.cfg | 9 +-------- qiita_ware/ebi.py | 19 ++++++++---------- qiita_ware/test/test_ebi.py | 25 +++--------------------- scripts/qiita-test-install | 6 ++---- 5 files changed, 14 insertions(+), 52 deletions(-) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index dc5a6eb60..d24fd0728 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -78,16 +78,12 @@ class ConfigurationManager(object): smtp_email : str The email address that mail will be sent from when sending mail from the SMTP server - ebi_access_key : str - The access key issued by EBI for REST submissions ebi_seq_xfer_user : str The user to use when submitting to EBI ebi_seq_xfer_pass : str The password for the ebi_seq_xfer_user ebi_seq_xfer_url : str The URL of EBI's sequence portal site - ebi_skip_curl_cert : bool - Whether or not to skip the certificate check when curling the metadata ebi_center_name : str The name of the sequencing center to use when doing EBI submissions ebi_organization_prefix : str @@ -213,14 +209,11 @@ def _get_smtp(self, config): def _get_ebi(self, config): sec_get = partial(config.get, 'ebi') - sec_getbool = partial(config.getboolean, 'ebi') - self.ebi_access_key = sec_get('EBI_ACCESS_KEY') self.ebi_seq_xfer_user = sec_get('EBI_SEQ_XFER_USER') self.ebi_seq_xfer_pass = sec_get('EBI_SEQ_XFER_PASS') self.ebi_seq_xfer_url = sec_get('EBI_SEQ_XFER_URL') self.ebi_dropbox_url = sec_get('EBI_DROPBOX_URL') - self.ebi_skip_curl_cert = sec_getbool('EBI_SKIP_CURL_CERT') self.ebi_center_name = sec_get('EBI_CENTER_NAME') self.ebi_organization_prefix = sec_get('EBI_ORGANIZATION_PREFIX') diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index b87f67264..e50e8951a 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -99,9 +99,6 @@ ADMIN_PASSWORD = # ----------------------------- EBI settings ----------------------------- [ebi] -# The access key issued by EBI for REST submissions -EBI_ACCESS_KEY = - # The user to use when submitting to EBI EBI_SEQ_XFER_USER = Webin-41528 @@ -117,12 +114,8 @@ EBI_SEQ_XFER_URL = webin.ebi.ac.uk # testing URL EBI_DROPBOX_URL = https://www-test.ebi.ac.uk/ena/submit/drop-box/submit/ -# Skip acertificate check when calling curl (necessary for using the EBI test -# dropbox URL) -EBI_SKIP_CURL_CERT = True - # The name of the sequencing center to use when doing EBI submissions -EBI_CENTER_NAME = CCME-COLORADO +EBI_CENTER_NAME = qiita-test # This string (with an underscore) will be prefixed to your EBI submission and # study aliases diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 13cce9319..8e8739c1a 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -1,6 +1,7 @@ from os.path import basename, join, isdir, isfile from os import makedirs, remove, listdir from datetime import date, timedelta +from urllib import quote from xml.etree import ElementTree as ET from xml.etree.ElementTree import ParseError from xml.sax.saxutils import escape @@ -623,8 +624,7 @@ def write_xml_file(self, element, fp): def generate_curl_command( self, ebi_seq_xfer_user=qiita_config.ebi_seq_xfer_user, - ebi_access_key=qiita_config.ebi_access_key, - ebi_skip_curl_cert=qiita_config.ebi_skip_curl_cert, + ebi_seq_xfer_pass=qiita_config.ebi_seq_xfer_pass, ebi_dropbox_url=qiita_config.ebi_dropbox_url): """Generates the curl command for submission @@ -632,10 +632,8 @@ def generate_curl_command( ---------- ebi_seq_xfer_user : str The user to use when submitting to EBI - ebi_access_key : str - The access key issued by EBI for REST submissions - ebi_skip_curl_cert : bool - If the curl certificate should be skipped + ebi_seq_xfer_pass : str + The user password issued by EBI for REST submissions ebi_dropbox_url : str The dropbox url @@ -650,13 +648,12 @@ def generate_curl_command( be generated before executing this function """ # make sure that the XML files have been generated - url = '?auth=ENA%20{0}%20{1}'.format(ebi_seq_xfer_user, - ebi_access_key) + url = '?auth=ENA%20{0}%20{1}'.format(quote(ebi_seq_xfer_user), + quote(ebi_seq_xfer_pass)) curl_command = ( - 'curl {0}-F "SUBMISSION=@{1}" -F "STUDY=@{2}" -F "SAMPLE=@{3}" ' - '-F "RUN=@{4}" -F "EXPERIMENT=@{5}" "{6}"' + 'curl -k -F "SUBMISSION=@{0}" -F "STUDY=@{1}" -F "SAMPLE=@{2}" ' + '-F "RUN=@{3}" -F "EXPERIMENT=@{4}" "{5}"' ).format( - '-k ' if ebi_skip_curl_cert else '', self.submission_xml_fp, self.study_xml_fp, self.sample_xml_fp, diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 395de01c3..87311a156 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -215,14 +215,11 @@ def test_generate_curl_command(self): submission = EBISubmission(2, 'ADD') test_ebi_seq_xfer_user = 'ebi_seq_xfer_user' - test_ebi_access_key = 'ebi_access_key' + test_ebi_seq_xfer_pass = 'ebi_seq_xfer_pass' test_ebi_dropbox_url = 'ebi_dropbox_url' - # Without curl certificate authentication - test_ebi_skip_curl_cert = True obs = submission.generate_curl_command(test_ebi_seq_xfer_user, - test_ebi_access_key, - test_ebi_skip_curl_cert, + test_ebi_seq_xfer_pass, test_ebi_dropbox_url) exp = ('curl -k ' '-F "SUBMISSION=@%(xml_dir)s/submission.xml" ' @@ -231,23 +228,7 @@ def test_generate_curl_command(self): '-F "RUN=@%(xml_dir)s/run.xml" ' '-F "EXPERIMENT=@%(xml_dir)s/experiment.xml" ' '"ebi_dropbox_url/?auth=ENA%%20ebi_seq_xfer_user' - '%%20ebi_access_key"') % {'xml_dir': submission.xml_dir} - self.assertEqual(obs, exp) - - # With curl certificate authentication - test_ebi_skip_curl_cert = False - obs = submission.generate_curl_command(test_ebi_seq_xfer_user, - test_ebi_access_key, - test_ebi_skip_curl_cert, - test_ebi_dropbox_url) - exp = ('curl ' - '-F "SUBMISSION=@%(xml_dir)s/submission.xml" ' - '-F "STUDY=@%(xml_dir)s/study.xml" ' - '-F "SAMPLE=@%(xml_dir)s/sample.xml" ' - '-F "RUN=@%(xml_dir)s/run.xml" ' - '-F "EXPERIMENT=@%(xml_dir)s/experiment.xml" ' - '"ebi_dropbox_url/?auth=ENA%%20ebi_seq_xfer_user' - '%%20ebi_access_key"') % {'xml_dir': submission.xml_dir} + '%%20ebi_seq_xfer_pass"') % {'xml_dir': submission.xml_dir} self.assertEqual(obs, exp) diff --git a/scripts/qiita-test-install b/scripts/qiita-test-install index 694ee8d57..3a01e1ff3 100755 --- a/scripts/qiita-test-install +++ b/scripts/qiita-test-install @@ -356,14 +356,12 @@ def main(): send_email = True except: send_email = False - ebi_credentials = (qiita_config.ebi_access_key != '' and - qiita_config.ebi_center_name != '' and + ebi_credentials = (qiita_config.ebi_center_name != '' and qiita_config.ebi_dropbox_url != '' and qiita_config.ebi_organization_prefix != '' and qiita_config.ebi_seq_xfer_pass != '' and qiita_config.ebi_seq_xfer_url != '' and - qiita_config.ebi_seq_xfer_user != '' and - qiita_config.ebi_skip_curl_cert != '') + qiita_config.ebi_seq_xfer_user != '') vamps_credentials = (qiita_config.vamps_pass != '' and qiita_config.vamps_url != '' and qiita_config.vamps_user != '') From 72cac32bad54fd74cc6420bfdaa5ea12bde7889a Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 6 Oct 2015 13:34:19 -0600 Subject: [PATCH 04/16] changing conf file and fixing setup_acsp.sh --- .../support_files/config_test_travis.cfg.enc | Bin 4000 -> 3776 bytes setup_acsp.sh | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qiita_core/support_files/config_test_travis.cfg.enc b/qiita_core/support_files/config_test_travis.cfg.enc index f247feb94d78a422403cf6734f7695c6191dcab1..8163844591c036e34bbada36f791482c6f436b09 100644 GIT binary patch literal 3776 zcmV;x4nOfn-V3(`3;c{ilIJoW_j2Cdz~e-}rV#WN8)2PFL?!ua!fc>T8^nk+*`k`& zxvBcAFDE2F#6TZ}}~4u%!}qflR}L3&A^u)U%O z-h6~E4B)z9)U4Tf^(Cl0)4zJ8e4JE_6}Yxk3*+&0-zF+2t7?(<5WS-0zu9b$0;83` zvG18&;+FvPf~_A8pP1S&W>H~+;8H+S1CjSw>I17M3MzS5oF;8}VHD|bIXOq;UI`zJ z0g7`OXqwk5+WvIqvcx`!uth*znJxuJ_R*hHje$CL(c%=KuCufsN*y(%wr`4e0io}H+-i+8m zhu=q(yQe+YHaD*Zo~v%ytov;t74~$@l$oXXXI{xWt;y2U*l&AZw#j&D{9h3JzXjUv z@GgGc+6YZ}X}8x8q`l{Brc<5}Xa;G!qLi3m zuH%G@+~;@@(lwrQY1@7h7`J(liv4?^^FFbp9&kLD!u@_d!(Ky4iQ7ONhE+=cUQ%=P z71TDzF#YHa-&wdZCF-f?7FEdme0U8o521d+F0Jx~Ir1za`4o3wlkSST4tcob{mb3R ziTGf3V#Rv~L$Dc0OYC@P+l=7FLfnM|o2zs7#6X*vIQPp&Y91JfG5G8PLXSUd=K{>g z#a8AUc!8*nX4{Zk@YtA**zZ(U{=|uPW9gJAF$BnJH9h^EhL=p~Ls_EoI# z_g&nfvYgtazB)RaaFPqlBb|by2A`II^-xfSjQM&;#-o4R;R40~tv93%l3BICj((#S zfOp}7MQL&4E!qJ1G%43jVmRbNj=mu?5*pHZmd$}uzpHI47A>0)$|LfFr zmw~l2&HdlNwk)_3EitMh{OTq~CiYkG73D$D*vxxR-PNmm?4cpAMqTPSlP_7l=ryw7 z38we%c*sz0#bVO_oAu>O?s^`F^)#?fKx{3)*gm8LN7II(K#_O4iz2U|C)KvD%Trbk zdO$Ps6e(=8KjUmMAw<_)mekI`d+D)!c2I1|&J^y^oq+*vsLHcwvf_BkXP1_D7imx;@rEOFc!bn>fiJ4G*nCA zr|ITP9VFZwFRjZ7Xj-{olcr9AgnQe~c9Qz zVC&xE>YvSqYI#QuoKDV;Ko6i-NT{RuyYHS(kS>WXZ0zFY9*Vdq`U^&Q z)X7`aLgD%|u1TWAY-9D_eQS}(@Nn4*u76Iw;QA-493IzWmm<;PfBBj zZzK=;kBB>R<#%XjT=*&a%ak&OB z3iPH$s7Zh*h^BZ+@$C}>gb3=zw%{i(@QM_3K1b=ADx}53@h$eINCSaqUkwd z7f>niPy}-rP|H+%Wy(H2-whwgDoFz<=2A5nUfCISEj#OgonZyTA#i5OE$Rab(bV`8 zg3Y-lCj{yNHTqVaO144CX{smF8J4hd+;Y%BYPHrrreS1ZzmH?K@Lu6gb%eb-Ep9Ca-(u{{Igr4>OcLqFo$@b?> zjm~sj398)AZ(5K-5O)-wSuMCORr}C)f}Da<*+4Ev+^DG8$)ZSYr4vRaYSHFc*PfPI zi5f}Bp{7fdSTasq$U`2z<;{VvEQFAvb$(qFe3ji~M;#WlU ze*lkcnr6nq1N=>-b$XFX%CeW`2DZrbYr869RNOX7&0V#A%YwyjjZQ@v!_huv_>xiH z`DQ8&iAgXFBd%(hcpkaatp%BIEcYBhy0G}yftmP?{Z{)9a1)-HEPojT!#vvez3O(% zQ`6v@-BkwAetPOjnE+5OKiZjF$h5zO$zl6zyvm?eR!(omb8Y$>P7OS6{ToP+O`y*l zqZ0pl_&LXTy%6^pt7OwolQu|isEc{hDhU{}inmTUF0D%+>7lps3~#O(I`iV?8;dMn zMxjopf=4KoE?UJAB+1%rQYrb>Q1?Oo=YgHMTmzR&I4-RIibS_XvRSXoU+0q+aH|W9 zJAH4NWr9dB2!)x2Z8}yknt^X%QO**+?C1bkFvGIKONhv+Oo0E1v*^064yx-;)^v{9 z2c+&8qT>iuiZso(;JtBS(_=BfHLm*7w0M*Q!Dud6h&sEBm(r7IJoQG73Fs&qXmF{{ z%!1jmpr?2j>_G^_W-NGY^kU#-E^|XwCF|DdbfC}#2k;Oeo{*O!`3v6#c0l_XiHpX? z87wf54nO>j%O|(nZ!ZxkUM}2K>fBcSk=NXXrCm@lnDbnJz$497OhWol-9dKPw%Ds} zz}BGilNYS)X2We_p|p5IHZ6+F$&iOY@LXT3#w>rS2wS<~WKTOF%R5CYah~UBK9)yJ!n0wB@ zxvz0;n)Z%?&o~o%!jhR5XX65rCF5q*5+1(B?R>k@kBX)D0t$Ek#M+?EhYqAwOO(>4 zv5^k6Nt7#Nrb#-L+GEcq8zq&#K(+SBF8JCo)p)u#;CyAFYwT-2&N@9|R8O!~W!om&kL_7$JaN&r2 zsV+L38k|l*t+fiwWqiUMEcjXACE@DT?G%P{ibn;2oI?PFmY$#iF7LQZYezxkHGWux zxR1R;vt0I~`E3?WY{P4#dDg?+M_WDWS-k&wzF-r1`%svB(bgO}q8)1JBH+w}Zm%xg zaliSZBFf%XG5IlL6BgUhTogyh3{m!&FPDR|DvM#`32LK~bT}M+p8aO$h&7*eSSdQV zBB%R)_}DvO4Epu$KOL0}-Y)NdSr~Ek>p$)(^ zdgqkbhq~gs3!eiJ%z2O%-(2!`xcz&X`R&Tx;aUxBl2ZYdu8!`Hw!M+;I zq{?pc6dA3*X8oS}J&I5cCa`(>Yq$**0$%@U1GkE^*|w|+_%~CpPHx;ReG433957R} z0+c{+uW^c>^HW$`NrR*8SU$pN*wzs77Z+}Is6m)|_> z8|13uzlIF0?L~=1E2zuWQ&+tw4Say;puX`vL@gPryE|UKI(iC=kXF)!+H*fPNPPSJ qm}WL7r&P=ryD*6?D4=Bdxdne&xHtgJrc30eiobW4CJlFZ;`KCgVxENsMJ{qe+PY%4cawuzuxD6m3YOa^e)`akR zl!yOf%nhSKZg~=&DBB+|M(DNfhg9pCL}{fxCAOpIZ}FBEKE|>SxIvvg#iCsFP^ycc zD2D*FbP0Eu4?b08o&i%VELy?wd*yCZUbeAw?7@`i?gcK;z&oLL^UU<=ZRMxbGVvlw_{cs;eN=wY$DhZ(>kI z6M+{uNlVvuaWq}YFIY{XHcV29o`4h<-K3N5d4TCl2$8%V?(=(XZjhK|jXlr))$)|{ za&B_Ar315a^j;4|uGuHNlrCZH1&oSd^-EAT#+J$(;}#O~0~xa%TH(P|zyo{zf(&*z zK}{fA*vrU*G_R#aVE#i0)8G!owZ^5dBo+f+yv&8F95~+yzsw0oi^{;jrb08G$snph z8cpOP^iSpBdAyU*KqpB2Qf;dMw^SXh`2$;)&&s~3Y2PLY9H=-gM#BcKHnEEJ#HWz& z$G2gtcrTSApe@}I#aBMUOomRhZ|fX$`IgLIJPzt6>P8enyVsiy=S7H6L(Zbi^lwP} z_O}&uEkb(LwBb~q(U?u%`_d6-Lq*+U1?d%c>Xv&@Si1ht*9ibZh{oXP?Vb1+xauPg z;`hq^gbqmvLUTWz*9EEtE_c;izk)C2_uc~A8Fv1v<%YL3xAy3#>cf~nOny1pp#GF^p^GgXkVR62EtH-^< zk3BBA`mNQGgquQ&V3yLyV+A^6k#`p@07-hAX_Nw4+!ItYY zHOScg@!quyt0Zs>7R8=E$hNOq`op;U9sg)6KZUR5sjoa#v`EE&jlaKI8Uthzm&+{ktK}vh zF0+WdS@=aq8Yyn%Mn(b11$DCequJ^>{gD5{**L);xPaZ;X^%`m{ zBUx(V1)z-2R)~V^80&lvkzTZK4V05IqKJ=2#RxwTUvSDuBio}RF-U4ocZ2g)t*4SL zp3P6T~u@T@2!1}Di-<^C`=+rs0vyEbIa=sozlp#V_lwZr<)o#Wc<;U zR1<#e=pdV@vUjk-c3^19G5ktR3tUzj5^zHUu}*SuQ0vc|nC2-XK4LRIbU}7Jj3#8o zop-yfnH=m=flkzVg|c!MGfa{*46Bx?uW|QPB^@@do>`-jhg>T2xV3L*2o)|)M@MOM-2iz-b#<%tk{Vn&`XVQDvEH@MKyvel zm4;H^9p9d&+h{MsY$g86}(*vP9&hlc`{_x;!{`^61{Jvin4CKDMSmXWh7D2WH(j zRx6|cce`m^jVC${tnh z)`MQx%xMb}CA)9AS;zsNOQiX69x3SR~0hlTN;>GI_@l5&pt z1QPg3);9r0rxFInONmSRoQ;vQ2d*6Hxs7y+u}SR!7W#q^viDsq?FuQ{9b}-&p()|7 zLr^GBhi}>_aP@*LF7;*z5#QD$Rp4%Tl__QdDjO~J+!LH;tm>jOF{#@!Zu3E6DPyh; zXhusLfI73}KI%73aT`&51K#YLVf)%E;iad=hPDg$T{oj?s`aB03KpL!)Bk=%8V z9!QVk{+NwSE?Fpko+p?#3dAlpZ_pzZ(!dFo)KC!+LLPm@UD*0fl2pPdrkgK?U^gX! z+wc6~f<%NGRGS~%vFT^PJsp5?$prbe*p%SL=u_U|r>eO?j`y{y?beP`1H&iPKqmeg zg%KYQC}?lT&#B2fDrJXi)ekui)jvHf>8-QmUPolo*lnbUXzxQOK#tuI-=gkRQ$g`C z{27{)q~%_-O=AjEK$kjs%Ua30$%r$BoODH1k#lchFtuoz?YX^pw`|3nH~ztS9gC z)EfP;rb@sQvHrD4Nid^ZyLQhdWE~kO_t%T*TNbgx4;~{tjut}-b~2&iub(bviHAwH zmMdvKVH^NBJ4^hNN8aDeh4G4t6Rz2d+UItM^m)87nm-MlY{O&Ke~pVzSI~&%T51C6 z@RCH>oA)H|iPF2xLaF6$g=3nyNlY5~>>yIt!2Q+Sz}YD?JlXK@p{u+K2n`FrplL~R zf$r;K>|{a#^9#q}_3m8?8OQ3IGk((GRdxHtvL3l<;Tp39?!Xzjp=W;NtncqVxmbHQ z`}gn{oy4lkaFt&&4QcL$1I9O(lJCs>_;xQrlcQ(0l&Db4EKFqS;R)QN%inmzUpzH+ zSxSFcwx$R3jH%3f`yX_0B0EEaG|^}@gtfTX!GZYfP1aw8 zQo_s{Y@%F&yHl$OqA%9AwnV_Pv0cEj580mdh4K=M`yv#!#Z0&L;cGzPBhhm?3mJ&` zd9n=?_VxImz%hP%t^c!J6ZWW>_(BV8Y7hu=J$;+>|B)0)5aBK4(|`m#6W=?3l*9Y< z2#$$={Fpi_v92*nCMX)Qr-;4$RV=P(>ZaXS-QOoAr4VqClo2CXmNtn%@@H5Y9SjDc zxNYzY!ZBCz0DM#{3!71p{Y?171#sd@=sDk(V`%RW?bLP6Zn#4mGXX94R5tr(N!)jH zxqWys6!^^x*z`zs_u5->;vWKq2rkd(%55$JW(;>K1r3>sbJ%!GAbCuef^oP|k(H?J zL|jzVd3up_4)^s7zku9d`z1~0{I81C*dPGhH{X+K*FCs>G#mc&`DTc>D&&ZSOcHz3 za|yvO#7k%JPL$x4jB3#0;#o$xGJBTw8MbHrhUG1hQaTbpo!G?h!Lt*xvUZC>w%gB= z+1}D6Ai;PKcYqPJI2%St`r?r9tH&DA!zH!u+s96S~r8U}9lc)xSKY|<-vK|aQsE2F6w1qUFZ zG%i5FqFXF1yatZP5PssrFWmi}>&p^AW2ASk0;cF~PEvroCpESlCn<%=ixAZ-eY*!U zhuJ5;b=nNQ^-2kOw{UxCiv`@e$2Q9w44TCmq-kB_ZJw$}CMiV6$@Y@MRtnvnHqQc?1DXa6L^B3|>b zk~gs78krwq`2Nz=bM&8-a zwJ}}g5_Q`37ahRSU?5J$OONsGjmA51j1=F=iQ@}E#4X~d>n44SdD4DF$e}*rg=@Jt z#jfB^)gc?}v`CiI8VGU`RX-sPUle^VRA~D`*0nHH7@J!yYneiiTlpgFw^XV3cDc}6 z=`St|*pbXLYtyPq8Vy|*`p05k9T|SA#Q@@b6dw#wHqr+=zvAH%8qBIw3B5>D)dgG& zfgP+vg1cP5eHvE$>|ubv0tjJDk^f1aKpg=qxBU5`8axB7y%;KC*pgSv4g6?TtqS?v zT$g4Kus diff --git a/setup_acsp.sh b/setup_acsp.sh index d1e88024f..c6a97a0f2 100644 --- a/setup_acsp.sh +++ b/setup_acsp.sh @@ -1,4 +1,4 @@ -openssl aes-256-cbc -K $encrypted_e698cf0e691c_key -iv $encrypted_e698cf0e691c_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d; \ -curl -s -O ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh +openssl aes-256-cbc -K $encrypted_e698cf0e691c_key -iv $encrypted_e698cf0e691c_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d +wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh -ascp-install-3.5.4.102989-linux-64-qiita.sh +./ascp-install-3.5.4.102989-linux-64-qiita.sh From c45be2eca7e5c837e1676b1c4ef2a0b76ac76805 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 8 Oct 2015 08:29:21 -0600 Subject: [PATCH 05/16] initial changes test_commands.py --- qiita_ware/commands.py | 70 +++++++++++++++++++------------------ qiita_ware/dispatchable.py | 6 +--- qiita_ware/ebi.py | 20 ++--------- qiita_ware/test/test_ebi.py | 2 ++ scripts/qiita | 4 +-- 5 files changed, 42 insertions(+), 60 deletions(-) diff --git a/qiita_ware/commands.py b/qiita_ware/commands.py index 9c9aba5d6..702c1fd48 100644 --- a/qiita_ware/commands.py +++ b/qiita_ware/commands.py @@ -10,6 +10,10 @@ from shutil import rmtree from tarfile import open as taropen from tempfile import mkdtemp +from os import environ +from traceback import format_exc +from subprocess import call +from shlex import split as shsplit from moi.job import system_call from qiita_db.study import Study @@ -19,11 +23,9 @@ from qiita_core.qiita_settings import qiita_config from qiita_ware.ebi import EBISubmission from qiita_ware.exceptions import ComputeError, EBISubmissionError -from traceback import format_exc -from os import environ -def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): +def submit_EBI(preprocessed_data_id, action, send): """Submit a preprocessed data to EBI Parameters @@ -34,27 +36,19 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): The action to perform with this data send : bool True to actually send the files - fastq_dir_fp : str, optional - The fastq filepath - - Notes - ----- - If fastq_dir_fp is passed, it must not contain any empty files, or - gzipped empty files """ # step 1: init and validate ebi_submission = EBISubmission(preprocessed_data_id, action) # step 2: generate demux fastq files - ebi_submission.preprocessed_data.update_insdc_status('demuxing samples') + ebi_submission.study.ebi_submission_status = 'submitting' try: ebi_submission.generate_demultiplexed_fastq() except: error_msg = format_exc() - if isdir(ebi_submission.ebi_dir): - rmtree(ebi_submission.ebi_dir) - ebi_submission.preprocessed_data.update_insdc_status( - 'failed: %s' % error_msg) + if isdir(ebi_submission.full_ebi_dir): + rmtree(ebi_submission.full_ebi_dir) + ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg LogEntry.create('Runtime', error_msg, info={'ebi_submission': preprocessed_data_id}) raise @@ -66,34 +60,42 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): # step 4: sending sequences old_ascp_pass = environ.get('ASPERA_SCP_PASS', '') environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass - seqs_cmds = ebi_submission.generate_send_sequences_cmd() + LogEntry.create('Runtime', ("Submitting sequences for pre_processed_id: " "%d" % preprocessed_data_id)) - try: - # place holder for moi call see #1477 - pass - except: - LogEntry.create('Fatal', seqs_cmds, - info={'ebi_submission': preprocessed_data_id}) - else: - LogEntry.create('Runtime', - ('Submission of sequences of pre_processed_id: ' - '%d completed successfully' % - preprocessed_data_id)) + for cmd in ebi_submission.generate_send_sequences_cmd(): + cmd_pieces = shsplit(cmd) + try: + call(cmd_pieces, stdout=open(ebi_submission.ascp_reply, 'a')) + except: + with open(ebi_submission.ascp_reply, 'r') as f: + content = f.read() + ebi_submission.study.ebi_submission_status = ( + "failed: ASCP - %s" % content) + LogEntry.create('Fatal', content, + info={ + 'ebi_submission': preprocessed_data_id, + 'fail': cmd}) + LogEntry.create('Runtime', + ('Submission of sequences of pre_processed_id: ' + '%d completed successfully' % + preprocessed_data_id)) environ['ASPERA_SCP_PASS'] = old_ascp_pass # step 5: sending xml and parsing answer - xmls_cmds = ebi_submission.generate_curl_command() + xmls_cmds = shsplit(ebi_submission.generate_curl_command()) LogEntry.create('Runtime', ("Submitting XMLs for pre_processed_id: " "%d" % preprocessed_data_id)) try: - # place holder for moi call see #1477 - xmls_cmds_moi = xmls_cmds + call(xmls_cmds, stdout=open(ebi_submission.curl_reply, 'w')) + with open(ebi_submission.curl_reply, 'r') as f: + xml_content = f.read() except: - # handle exception - LogEntry.create('Fatal', seqs_cmds, + with open(ebi_submission.curl_reply, 'r') as f: + xml_content = f.read() + LogEntry.create('Fatal', xml_content, info={'ebi_submission': preprocessed_data_id}) else: LogEntry.create('Runtime', @@ -103,10 +105,10 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): try: st_acc, sa_acc, bio_acc, ex_acc, run_acc = \ - ebi_submission.parse_EBI_reply(xmls_cmds_moi) + ebi_submission.parse_EBI_reply(xml_content) except EBISubmissionError as e: le = LogEntry.create( - 'Fatal', "Command: %s\nError: %s\n" % (xmls_cmds_moi, str(e)), + 'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)), info={'ebi_submission': preprocessed_data_id}) ebi_submission.preprocessed_data.update_insdc_status('failed') raise ComputeError("EBI Submission failed! Log id: %d" % le.id) diff --git a/qiita_ware/dispatchable.py b/qiita_ware/dispatchable.py index 0c5883541..f5690814d 100644 --- a/qiita_ware/dispatchable.py +++ b/qiita_ware/dispatchable.py @@ -52,11 +52,7 @@ def preprocessor(study_id, prep_template_id, param_id, param_constructor): def submit_to_ebi(preprocessed_data_id, submission_type): """Submit a study to EBI""" - study_acc, submission_acc = submit_EBI(preprocessed_data_id, - submission_type, - True) - - return study_acc, submission_acc + submit_EBI(preprocessed_data_id, submission_type, True) def submit_to_VAMPS(preprocessed_data_id): diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 3a1cf3de4..d1ce7b50c 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -23,24 +23,6 @@ from qiita_db.metadata_template import PrepTemplate, SampleTemplate -class InvalidMetadataError(Exception): - """Error that is raised when metadata is not representable as a string""" - pass - - -class SampleAlreadyExistsError(Exception): - """Error that is raised when a sample is added to a submission that already - has a sample by that name""" - pass - - -class NoXMLError(Exception): - """Error that is raised when the generation of one XML file cannot be - completed because it depends on another XML file that has not yet been - generated""" - pass - - def clean_whitespace(text): """Standardizes whitespaces so there is only one space separating tokens @@ -161,6 +143,8 @@ def __init__(self, preprocessed_data_id, action): _, base_fp = get_mountpoint("preprocessed_data")[0] self.ebi_dir = '%d_ebi_submission' % preprocessed_data_id self.full_ebi_dir = join(base_fp, self.ebi_dir) + self.ascp_reply = join(self.full_ebi_dir, 'ascp_reply.txt') + self.curl_reply = join(self.full_ebi_dir, 'curl_reply.xml') self.xml_dir = join(self.full_ebi_dir, 'xml_dir') self.study_xml_fp = None self.sample_xml_fp = None diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 265637453..50df08f37 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -78,6 +78,8 @@ def test_init(self): self.assertItemsEqual(e.pmids, ['123456', '7891011']) self.assertEqual(e.action, action) + self.assertEqual(e.ascp_reply, join(e.full_ebi_dir, 'ascp_reply.txt')) + self.assertEqual(e.curl_reply, join(e.full_ebi_dir, 'curl_reply.xml')) get_output_fp = partial(join, e.full_ebi_dir) self.assertEqual(e.xml_dir, get_output_fp('xml_dir')) self.assertIsNone(e.study_xml_fp) diff --git a/scripts/qiita b/scripts/qiita index 80251518f..4f4848906 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -354,10 +354,8 @@ def remove_study(portal, studies): @click.option('--send/--no-send', default=False, help="Controls whether or " "not sequence files and metadata will actually be sent to EBI " "(default is to generate all the files, but not to send)") -@click.option('--fastq-dir', type=click.Path(exists=True, file_okay=False, - resolve_path=True)) def submit(preprocessed_data_id, action, send, fastq_dir): - _submit_EBI(preprocessed_data_id, action, send, fastq_dir) + _submit_EBI(preprocessed_data_id, action, send) # ############################################################################# From 1928106241bd3d065a4355fc7aee96e109c2bf47 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 8 Oct 2015 15:25:57 -0600 Subject: [PATCH 06/16] test ready --- qiita_ware/commands.py | 3 +- qiita_ware/ebi.py | 25 ++-- qiita_ware/test/test_commands.py | 196 +++++++++++++++++++++++++++++++ qiita_ware/test/test_ebi.py | 20 ++-- 4 files changed, 221 insertions(+), 23 deletions(-) create mode 100644 qiita_ware/test/test_commands.py diff --git a/qiita_ware/commands.py b/qiita_ware/commands.py index 702c1fd48..6dad70d38 100644 --- a/qiita_ware/commands.py +++ b/qiita_ware/commands.py @@ -110,7 +110,8 @@ def submit_EBI(preprocessed_data_id, action, send): le = LogEntry.create( 'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)), info={'ebi_submission': preprocessed_data_id}) - ebi_submission.preprocessed_data.update_insdc_status('failed') + ebi_submission.study.ebi_submission_status = ( + "failed: XML submission, log id: %d" % le.id) raise ComputeError("EBI Submission failed! Log id: %d" % le.id) ebi_submission.study.ebi_submission_status = 'submitted' diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index d1ce7b50c..10e37f0cf 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -712,17 +712,18 @@ def generate_curl_command( # make sure that the XML files have been generated url = '?auth=ENA%20{0}%20{1}'.format(quote(ebi_seq_xfer_user), quote(ebi_seq_xfer_pass)) - curl_command = ( - 'curl -k -F "SUBMISSION=@{0}" -F "STUDY=@{1}" -F "SAMPLE=@{2}" ' - '-F "RUN=@{3}" -F "EXPERIMENT=@{4}" "{5}"' - ).format( - self.submission_xml_fp, - self.study_xml_fp, - self.sample_xml_fp, - self.run_xml_fp, - self.experiment_xml_fp, - join(ebi_dropbox_url, url) - ) + curl_command = 'curl -sS -k' + if self.submission_xml_fp is not None: + curl_command += ' -F "SUBMISSION=@%s"' % self.submission_xml_fp + if self.study_xml_fp is not None: + curl_command += ' -F "STUDY=@%s"' % self.study_xml_fp + if self.sample_xml_fp is not None: + curl_command += ' -F "SAMPLE=@%s"' % self.sample_xml_fp + if self.run_xml_fp is not None: + curl_command += ' -F "RUN=@%s"' % self.run_xml_fp + if self.experiment_xml_fp is not None: + curl_command += ' -F "EXPERIMENT=@%s"' % self.experiment_xml_fp + curl_command += ' "' + join(ebi_dropbox_url, url) + '"' return curl_command @@ -744,7 +745,7 @@ def generate_send_sequences_cmd(self): fastqs_div = [fastqs[i::10] for i in range(10) if fastqs[i::10]] ascp_commands = [] for f in fastqs_div: - ascp_commands.append('ascp --ignore-host-key -L- -d -QT -k2 ' + ascp_commands.append('ascp --ignore-host-key -d -QT -k2 ' '{0} {1}@{2}:./{3}/'.format( ' '.join(f), qiita_config.ebi_seq_xfer_user, diff --git a/qiita_ware/test/test_commands.py b/qiita_ware/test/test_commands.py new file mode 100644 index 000000000..6aa3e8404 --- /dev/null +++ b/qiita_ware/test/test_commands.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python + +from __future__ import division + +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- +from unittest import TestCase, main +from os.path import join +from warnings import simplefilter +from tempfile import mkdtemp +import pandas as pd +from datetime import datetime + +from h5py import File + +from qiita_ware.exceptions import ComputeError +from qiita_ware.demux import to_hdf5 +from qiita_ware.commands import submit_EBI +from qiita_db.data import PreprocessedData +from qiita_db.study import Study, StudyPerson +from qiita_db.metadata_template import PrepTemplate, SampleTemplate +from qiita_db.user import User +from qiita_core.util import qiita_test_checker + + +@qiita_test_checker() +class CommandsTests(TestCase): + def setUp(self): + self.files_to_remove = [] + self.temp_dir = mkdtemp() + self.files_to_remove.append(self.temp_dir) + + def write_demux_files(self, prep_template, generate_hdf5=True): + """Writes a demux test file to avoid duplication of code""" + fna_fp = join(self.temp_dir, 'seqs.fna') + demux_fp = join(self.temp_dir, 'demux.seqs') + if generate_hdf5: + with open(fna_fp, 'w') as f: + f.write(FASTA_EXAMPLE) + with File(demux_fp, "w") as f: + to_hdf5(fna_fp, f) + else: + with open(demux_fp, 'w') as f: + f.write('') + + ppd = PreprocessedData.create(Study(1), + "preprocessed_sequence_illumina_params", + 1, [(demux_fp, 6)], prep_template) + return ppd + + def generate_new_study_with_preprocessed_data(self): + """Creates a new study up to the processed data for testing""" + # ignoring warnings generated when adding templates + simplefilter("ignore") + info = { + "timeseries_type_id": 1, + "metadata_complete": True, + "mixs_compliant": True, + "number_samples_collected": 3, + "number_samples_promised": 3, + "study_alias": "Test EBI", + "study_description": "Study for testing EBI", + "study_abstract": "Study for testing EBI", + "emp_person_id": StudyPerson(2), + "principal_investigator_id": StudyPerson(3), + "lab_person_id": StudyPerson(1) + } + study = Study.create(User('test@foo.bar'), "Test EBI study", [1], info) + metadata_dict = { + 'Sample1': {'collection_timestamp': datetime(2015, 6, 1, 7, 0, 0), + 'physical_specimen_location': 'location1', + 'taxon_id': 9606, + 'scientific_name': 'homo sapiens', + 'Description': 'Test Sample 1'}, + 'Sample2': {'collection_timestamp': datetime(2015, 6, 2, 7, 0, 0), + 'physical_specimen_location': 'location1', + 'taxon_id': 9606, + 'scientific_name': 'homo sapiens', + 'Description': 'Test Sample 2'}, + 'Sample3': {'collection_timestamp': datetime(2015, 6, 3, 7, 0, 0), + 'physical_specimen_location': 'location1', + 'taxon_id': 9606, + 'scientific_name': 'homo sapiens', + 'Description': 'Test Sample 3'} + } + metadata = pd.DataFrame.from_dict(metadata_dict, orient='index') + SampleTemplate.create(metadata, study) + metadata_dict = { + 'Sample1': {'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'CGTAGAGCTCTC', + 'center_name': 'KnightLab', + 'platform': 'ILLUMINA', + 'instrument_model': 'Illumina MiSeq', + 'library_construction_protocol': 'Protocol ABC', + 'experiment_design_description': "Random value 1"}, + 'Sample2': {'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'CGTAGAGCTCTA', + 'center_name': 'KnightLab', + 'platform': 'ILLUMINA', + 'instrument_model': 'Illumina MiSeq', + 'library_construction_protocol': 'Protocol ABC', + 'experiment_design_description': "Random value 2"}, + 'Sample3': {'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'CGTAGAGCTCTT', + 'center_name': 'KnightLab', + 'platform': 'ILLUMINA', + 'instrument_model': 'Illumina MiSeq', + 'library_construction_protocol': 'Protocol ABC', + 'experiment_design_description': "Random value 3"}, + } + metadata = pd.DataFrame.from_dict(metadata_dict, orient='index') + pt = PrepTemplate.create(metadata, study, "16S", 'Metagenomics') + fna_fp = join(self.temp_dir, 'seqs.fna') + demux_fp = join(self.temp_dir, 'demux.seqs') + with open(fna_fp, 'w') as f: + f.write(FASTA_EXAMPLE_2.format(study.id)) + with File(demux_fp, 'w') as f: + to_hdf5(fna_fp, f) + + ppd = PreprocessedData.create( + study, "preprocessed_sequence_illumina_params", 1, + [(demux_fp, 6)], pt) + + return ppd + + def test_submit_EBI_step_2_failure(self): + ppd = self.write_demux_files(PrepTemplate(1), False) + # + # with self.assertRaises(AttributeError): + # submit_EBI(ppd.id, 'ADD', True) + + def test_submit_EBI_parse_EBI_reply_failure(self): + ppd = self.write_demux_files(PrepTemplate(1)) + with self.assertRaises(ComputeError): + submit_EBI(ppd.id, 'ADD', True) + + def test_full_submission(self): + ppd = self.generate_new_study_with_preprocessed_data() + # + # submit_EBI(ppd.id, 'ADD', True) + + +FASTA_EXAMPLE = """>1.SKB2.640194_1 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB2.640194_2 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB2.640194_3 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM4.640180_4 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM4.640180_5 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB3.640195_6 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB6.640176_7 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKD6.640190_8 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM6.640187_9 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKD9.640182_10 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM8.640201_11 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM2.640199_12 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +""" + +FASTA_EXAMPLE_2 = """>{0}.Sample1_1 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample1_2 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample1_3 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample2_4 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample2_5 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample2_6 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample3_7 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample3_8 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample3_9 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +""" + + +if __name__ == '__main__': + main() diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 50df08f37..16ac87388 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -232,7 +232,7 @@ def test_generate_curl_command(self): obs = submission.generate_curl_command(test_ebi_seq_xfer_user, test_ebi_seq_xfer_pass, test_ebi_dropbox_url) - exp = ('curl -k ' + exp = ('curl -sS -k ' '-F "SUBMISSION=@/some/path/submission.xml" ' '-F "STUDY=@/some/path/study.xml" ' '-F "SAMPLE=@/some/path/sample.xml" ' @@ -599,31 +599,31 @@ def test_generate_send_sequences_cmd(self): e.generate_xml_files() obs = e.generate_send_sequences_cmd() _, base_fp = get_mountpoint("preprocessed_data")[0] - exp = ('ascp --ignore-host-key -L- -d -QT -k2 ' + exp = ('ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKB2.640194.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKM4.640180.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKB3.640195.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKB6.640176.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKD6.640190.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKM6.640187.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKD9.640182.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKM8.640201.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' - 'ascp --ignore-host-key -L- -d -QT -k2 ' + 'ascp --ignore-host-key -d -QT -k2 ' '%(ebi_dir)s/1.SKM2.640199.fastq.gz ' 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/' % { 'ebi_dir': e.full_ebi_dir}).split('\n') From 6417aa6ad74b9bc109dce473722d271a9c32b115 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 9 Oct 2015 08:14:45 -0600 Subject: [PATCH 07/16] fixing last issues --- .../source/tutorials/prepare-templates.rst | 4 +- qiita_ware/ebi.py | 69 ++++++++++++------- qiita_ware/test/test_commands.py | 6 +- qiita_ware/test/test_ebi.py | 37 ++++++++-- 4 files changed, 81 insertions(+), 35 deletions(-) diff --git a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst index cc84b3ea1..700bd942a 100644 --- a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst +++ b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst @@ -37,6 +37,8 @@ These are the columns required for successfully submit your data to EBI: | ``physical_specimen_location`` | free text | Where you would go to find physical sample or DNA, regardless of whether it is still available or not. | +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ | ``taxon_id`` | integer | NCBI's taxon id for the sample | ++-------------------------------------+-------------------------------------------+----------------------------------------------------------------------------------+ +| ``description`` | free text | Description of the sample. | +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ | ``scientific_name`` | free text | NCBI's scientific name for the provided taxon ID | +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ @@ -96,8 +98,6 @@ These are the columns required for successfully submit your data to EBI: +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``library_construction_protocol`` | free text | Brief description or reference to the protocol that was used for preparing this amplicon library starting from DNA, usually this includes what genomic region was targeted such as *16S*, *ITS*, *18S*, etc. | +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``description`` | free text | Description of the sample. | -+-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``experiment_design_description`` | free text | High-level description of the study (for example, *A longitudinal study of the gut microbiome of two human subjects*). | +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 10e37f0cf..3e54d5677 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -712,20 +712,20 @@ def generate_curl_command( # make sure that the XML files have been generated url = '?auth=ENA%20{0}%20{1}'.format(quote(ebi_seq_xfer_user), quote(ebi_seq_xfer_pass)) - curl_command = 'curl -sS -k' + curl_cmd = ['curl -sS -k'] if self.submission_xml_fp is not None: - curl_command += ' -F "SUBMISSION=@%s"' % self.submission_xml_fp + curl_cmd.append(' -F "SUBMISSION=@%s"' % self.submission_xml_fp) if self.study_xml_fp is not None: - curl_command += ' -F "STUDY=@%s"' % self.study_xml_fp + curl_cmd.append(' -F "STUDY=@%s"' % self.study_xml_fp) if self.sample_xml_fp is not None: - curl_command += ' -F "SAMPLE=@%s"' % self.sample_xml_fp + curl_cmd.append(' -F "SAMPLE=@%s"' % self.sample_xml_fp) if self.run_xml_fp is not None: - curl_command += ' -F "RUN=@%s"' % self.run_xml_fp + curl_cmd.append(' -F "RUN=@%s"' % self.run_xml_fp) if self.experiment_xml_fp is not None: - curl_command += ' -F "EXPERIMENT=@%s"' % self.experiment_xml_fp - curl_command += ' "' + join(ebi_dropbox_url, url) + '"' + curl_cmd.append(' -F "EXPERIMENT=@%s"' % self.experiment_xml_fp) + curl_cmd.append(' "%s"' % join(ebi_dropbox_url, url)) - return curl_command + return ''.join(curl_cmd) def generate_send_sequences_cmd(self): """Generate the sequences to EBI via ascp command @@ -863,6 +863,12 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): those cases where the fastq.gz files weren't written or exist. This is an indication that they had no sequences and this kind of files are not accepted in EBI + + Raises + ------ + EBISubmissionError + - The demux file couldn't be read + - All samples are removed """ ppd = self.preprocessed_data @@ -879,24 +885,29 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): if ftype == 'preprocessed_demux'][0] demux_samples = set() - with open_file(demux) as demux_fh: - for s, i in to_per_sample_ascii(demux_fh, - self.prep_template.keys()): - sample_fp = self.sample_demux_fps[s] - wrote_sequences = False - with GzipFile(sample_fp, mode='w', mtime=mtime) as fh: - for record in i: - fh.write(record) - wrote_sequences = True - - if wrote_sequences: - demux_samples.add(s) - else: - del(self.samples[s]) - del(self.samples_prep[s]) - del(self.sample_demux_fps[s]) - remove(sample_fp) - + try: + with open_file(demux) as demux_fh: + for s, i in to_per_sample_ascii(demux_fh, + self.prep_template.keys()): + sample_fp = self.sample_demux_fps[s] + wrote_sequences = False + with GzipFile(sample_fp, mode='w', mtime=mtime) as fh: + for record in i: + fh.write(record) + wrote_sequences = True + + if wrote_sequences: + demux_samples.add(s) + else: + del(self.samples[s]) + del(self.samples_prep[s]) + del(self.sample_demux_fps[s]) + remove(sample_fp) + except Exception as e: + error_msg = ("There was an error parsing the demux file: " + "%s" % str(e)) + LogEntry.create('Runtime', error_msg) + raise EBISubmissionError(error_msg) else: demux_samples = set() extension = '.fastq.gz' @@ -913,4 +924,10 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): del(self.samples_prep[ms]) del(self.sample_demux_fps[ms]) + if not demux_samples: + error_msg = ("All samples were removed from the submission " + "because the demux file is empty or the sample names" + "do not match.") + LogEntry.create('Runtime', error_msg) + raise EBISubmissionError(error_msg) return demux_samples diff --git a/qiita_ware/test/test_commands.py b/qiita_ware/test/test_commands.py index 6aa3e8404..ed9de3f68 100644 --- a/qiita_ware/test/test_commands.py +++ b/qiita_ware/test/test_commands.py @@ -129,7 +129,8 @@ def generate_new_study_with_preprocessed_data(self): return ppd def test_submit_EBI_step_2_failure(self): - ppd = self.write_demux_files(PrepTemplate(1), False) + pass + # ppd = self.write_demux_files(PrepTemplate(1), False) # # with self.assertRaises(AttributeError): # submit_EBI(ppd.id, 'ADD', True) @@ -140,7 +141,8 @@ def test_submit_EBI_parse_EBI_reply_failure(self): submit_EBI(ppd.id, 'ADD', True) def test_full_submission(self): - ppd = self.generate_new_study_with_preprocessed_data() + pass + # ppd = self.generate_new_study_with_preprocessed_data() # # submit_EBI(ppd.id, 'ADD', True) diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 16ac87388..25bd32641 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -245,14 +245,26 @@ def test_generate_curl_command(self): @qiita_test_checker() class TestEBISubmissionWriteRead(TestEBISubmission): - def write_demux_files(self, prep_template): + def write_demux_files(self, prep_template, sequences='FASTA-EXAMPLE'): """Writes a demux test file to avoid duplication of code""" fna_fp = join(self.temp_dir, 'seqs.fna') demux_fp = join(self.temp_dir, 'demux.seqs') - with open(fna_fp, 'w') as f: - f.write(FASTA_EXAMPLE) - with File(demux_fp, "w") as f: - to_hdf5(fna_fp, f) + if sequences == 'FASTA-EXAMPLE': + with open(fna_fp, 'w') as f: + f.write(FASTA_EXAMPLE) + with File(demux_fp, "w") as f: + to_hdf5(fna_fp, f) + elif sequences == 'WRONG-SEQS': + with open(fna_fp, 'w') as f: + f.write('>a_1 X orig_bc=X new_bc=X bc_diffs=0\nCCC') + with File(demux_fp, "w") as f: + to_hdf5(fna_fp, f) + elif sequences == 'EMPTY': + with open(demux_fp, 'w') as f: + f.write("") + else: + raise ValueError('Wrong sequences values: %s. Valid values: ' + 'FASTA_EXAMPLE, WRONG-SEQS, EMPTY' % sequences) ppd = PreprocessedData.create(Study(1), "preprocessed_sequence_illumina_params", @@ -561,6 +573,21 @@ def test_generate_xml_files(self): self.assertIsNone(e.study_xml_fp) self.assertIsNotNone(e.submission_xml_fp) + def test_generate_demultiplexed_fastq_failure(self): + # generating demux file for testing + ppd = self.write_demux_files(PrepTemplate(1), 'EMPTY') + + ebi_submission = EBISubmission(ppd.id, 'ADD') + self.files_to_remove.append(ebi_submission.full_ebi_dir) + with self.assertRaises(EBISubmissionError): + ebi_submission.generate_demultiplexed_fastq() + + ppd = self.write_demux_files(PrepTemplate(1), 'WRONG-SEQS') + ebi_submission = EBISubmission(ppd.id, 'ADD') + self.files_to_remove.append(ebi_submission.full_ebi_dir) + with self.assertRaises(EBISubmissionError): + ebi_submission.generate_demultiplexed_fastq() + def test_generate_demultiplexed_fastq(self): # generating demux file for testing exp_demux_samples = set( From abc5fe81a144e190380eedb50d9f26892f809325 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 9 Oct 2015 12:02:06 -0600 Subject: [PATCH 08/16] addressing @ElDeveloper and @mortonjt comments --- .../doc/source/tutorials/prepare-templates.rst | 2 +- qiita_ware/ebi.py | 2 +- qiita_ware/test/test_commands.py | 8 ++++---- qiita_ware/test/test_ebi.py | 2 -- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst index 700bd942a..7a8b675ee 100644 --- a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst +++ b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst @@ -37,7 +37,7 @@ These are the columns required for successfully submit your data to EBI: | ``physical_specimen_location`` | free text | Where you would go to find physical sample or DNA, regardless of whether it is still available or not. | +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ | ``taxon_id`` | integer | NCBI's taxon id for the sample | -+-------------------------------------+-------------------------------------------+----------------------------------------------------------------------------------+ ++----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ | ``description`` | free text | Description of the sample. | +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ | ``scientific_name`` | free text | NCBI's scientific name for the provided taxon ID | diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 3e54d5677..75f55cab1 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -926,7 +926,7 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): if not demux_samples: error_msg = ("All samples were removed from the submission " - "because the demux file is empty or the sample names" + "because the demux file is empty or the sample names " "do not match.") LogEntry.create('Runtime', error_msg) raise EBISubmissionError(error_msg) diff --git a/qiita_ware/test/test_commands.py b/qiita_ware/test/test_commands.py index ed9de3f68..0642cf097 100644 --- a/qiita_ware/test/test_commands.py +++ b/qiita_ware/test/test_commands.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from __future__ import division # ----------------------------------------------------------------------------- @@ -129,11 +127,12 @@ def generate_new_study_with_preprocessed_data(self): return ppd def test_submit_EBI_step_2_failure(self): - pass + # see issue #406 # ppd = self.write_demux_files(PrepTemplate(1), False) # # with self.assertRaises(AttributeError): # submit_EBI(ppd.id, 'ADD', True) + pass def test_submit_EBI_parse_EBI_reply_failure(self): ppd = self.write_demux_files(PrepTemplate(1)) @@ -141,10 +140,11 @@ def test_submit_EBI_parse_EBI_reply_failure(self): submit_EBI(ppd.id, 'ADD', True) def test_full_submission(self): - pass + # see issue #406 # ppd = self.generate_new_study_with_preprocessed_data() # # submit_EBI(ppd.id, 'ADD', True) + pass FASTA_EXAMPLE = """>1.SKB2.640194_1 X orig_bc=X new_bc=X bc_diffs=0 diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 25bd32641..5b57b5ae6 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from __future__ import division # ----------------------------------------------------------------------------- From 13f80b147bcf0f12ed5b3af3affc5d2d2c007c5d Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 9 Oct 2015 14:02:03 -0600 Subject: [PATCH 09/16] addressing @josenavas comments --- qiita_ware/commands.py | 56 ++++++++++++++++++++++-------------------- qiita_ware/ebi.py | 56 +++++++++++++++++++----------------------- 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/qiita_ware/commands.py b/qiita_ware/commands.py index 6dad70d38..426cf3e4f 100644 --- a/qiita_ware/commands.py +++ b/qiita_ware/commands.py @@ -12,10 +12,8 @@ from tempfile import mkdtemp from os import environ from traceback import format_exc -from subprocess import call -from shlex import split as shsplit - from moi.job import system_call + from qiita_db.study import Study from qiita_db.data import PreprocessedData from qiita_db.metadata_template import PrepTemplate, SampleTemplate @@ -64,44 +62,50 @@ def submit_EBI(preprocessed_data_id, action, send): LogEntry.create('Runtime', ("Submitting sequences for pre_processed_id: " "%d" % preprocessed_data_id)) - for cmd in ebi_submission.generate_send_sequences_cmd(): - cmd_pieces = shsplit(cmd) - try: - call(cmd_pieces, stdout=open(ebi_submission.ascp_reply, 'a')) - except: - with open(ebi_submission.ascp_reply, 'r') as f: - content = f.read() - ebi_submission.study.ebi_submission_status = ( - "failed: ASCP - %s" % content) - LogEntry.create('Fatal', content, - info={ - 'ebi_submission': preprocessed_data_id, - 'fail': cmd}) + try: + for cmd in ebi_submission.generate_send_sequences_cmd(): + try: + stdout, stderr, _ = system_call(cmd) + except Exception as e: + error = str(e) + ebi_submission.study.ebi_submission_status = ( + "failed: ASCP - %s" % error) + LogEntry.create('Fatal', error, + info={ + 'ebi_submission': preprocessed_data_id, + 'fail': cmd}) + raise + finally: + open(ebi_submission.ascp_reply, 'a').write( + 'stdout:\n%s\n\nstderr: %s' % (stdout, stderr)) + finally: + environ['ASPERA_SCP_PASS'] = old_ascp_pass LogEntry.create('Runtime', ('Submission of sequences of pre_processed_id: ' '%d completed successfully' % preprocessed_data_id)) - environ['ASPERA_SCP_PASS'] = old_ascp_pass # step 5: sending xml and parsing answer - xmls_cmds = shsplit(ebi_submission.generate_curl_command()) + xmls_cmds = ebi_submission.generate_curl_command() LogEntry.create('Runtime', ("Submitting XMLs for pre_processed_id: " "%d" % preprocessed_data_id)) try: - call(xmls_cmds, stdout=open(ebi_submission.curl_reply, 'w')) - with open(ebi_submission.curl_reply, 'r') as f: - xml_content = f.read() - except: - with open(ebi_submission.curl_reply, 'r') as f: - xml_content = f.read() - LogEntry.create('Fatal', xml_content, - info={'ebi_submission': preprocessed_data_id}) + xml_content, stderr, _ = system_call(xmls_cmds) + + except Exception as e: + error = str(e) + LogEntry.create('Fatal', error, + info={'ebi_submission': preprocessed_data_id, + 'fail': cmd}) else: LogEntry.create('Runtime', ('Submission of sequences of pre_processed_id: ' '%d completed successfully' % preprocessed_data_id)) + finally: + open(ebi_submission.curl_reply, 'w').write( + 'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr)) try: st_acc, sa_acc, bio_acc, ex_acc, run_acc = \ diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 75f55cab1..e795443c6 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -7,7 +7,7 @@ from xml.sax.saxutils import escape from gzip import GzipFile from functools import partial - +from h5py import File from future.utils import viewitems, viewkeys from skbio.util import safe_md5, create_dir @@ -17,7 +17,7 @@ from qiita_ware.util import open_file from qiita_db.logger import LogEntry from qiita_db.ontology import Ontology -from qiita_db.util import convert_to_id, get_mountpoint, insert_filepaths +from qiita_db.util import convert_to_id, get_mountpoint from qiita_db.study import Study from qiita_db.data import PreprocessedData from qiita_db.metadata_template import PrepTemplate, SampleTemplate @@ -874,40 +874,34 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): dir_not_exists = not isdir(self.full_ebi_dir) if dir_not_exists or rewrite_fastq: - if dir_not_exists: - makedirs(self.full_ebi_dir) - filetype_id = convert_to_id('directory', 'filepath_type') - insert_filepaths([(self.full_ebi_dir, filetype_id)], -1, - 'preprocessed_data', 'filepath', - move_files=False) + makedirs(self.full_ebi_dir) demux = [path for _, path, ftype in ppd.get_filepaths() if ftype == 'preprocessed_demux'][0] demux_samples = set() - try: - with open_file(demux) as demux_fh: - for s, i in to_per_sample_ascii(demux_fh, - self.prep_template.keys()): - sample_fp = self.sample_demux_fps[s] - wrote_sequences = False - with GzipFile(sample_fp, mode='w', mtime=mtime) as fh: - for record in i: - fh.write(record) - wrote_sequences = True - - if wrote_sequences: - demux_samples.add(s) - else: - del(self.samples[s]) - del(self.samples_prep[s]) - del(self.sample_demux_fps[s]) - remove(sample_fp) - except Exception as e: - error_msg = ("There was an error parsing the demux file: " - "%s" % str(e)) - LogEntry.create('Runtime', error_msg) - raise EBISubmissionError(error_msg) + with open_file(demux) as demux_fh: + if not isinstance(demux_fh, File): + error_msg = ("There was an error parsing the demux file: " + "%s" % demux) + LogEntry.create('Runtime', error_msg) + raise EBISubmissionError(error_msg) + for s, i in to_per_sample_ascii(demux_fh, + self.prep_template.keys()): + sample_fp = self.sample_demux_fps[s] + wrote_sequences = False + with GzipFile(sample_fp, mode='w', mtime=mtime) as fh: + for record in i: + fh.write(record) + wrote_sequences = True + + if wrote_sequences: + demux_samples.add(s) + else: + del(self.samples[s]) + del(self.samples_prep[s]) + del(self.sample_demux_fps[s]) + remove(sample_fp) else: demux_samples = set() extension = '.fastq.gz' From 0586d8b4e633176195b4d190a6eb7579c524a621 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 9 Oct 2015 15:04:33 -0600 Subject: [PATCH 10/16] fixing error --- .travis.yml | 5 ++++- qiita_ware/commands.py | 7 ++++--- qiita_ware/ebi.py | 3 +-- setup_acsp.sh | 4 ---- 4 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 setup_acsp.sh diff --git a/.travis.yml b/.travis.yml index 3735fb9c0..4633b0344 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,10 @@ before_install: # Update conda itself - conda update --yes conda # Downloading and setting up ascp for EBI testing - - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then sh setup_acsp.sh; fi + - wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh + - chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh + - ./ascp-install-3.5.4.102989-linux-64-qiita.sh + - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then openssl aes-256-cbc -K $encrypted_e698cf0e691c_key -iv $encrypted_e698cf0e691c_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d; fi install: # install a few of the dependencies that pip would otherwise try to install # when intalling scikit-bio diff --git a/qiita_ware/commands.py b/qiita_ware/commands.py index 426cf3e4f..9bc2b8d56 100644 --- a/qiita_ware/commands.py +++ b/qiita_ware/commands.py @@ -67,10 +67,11 @@ def submit_EBI(preprocessed_data_id, action, send): try: stdout, stderr, _ = system_call(cmd) except Exception as e: - error = str(e) + stdout = '' + stderr = str(e) ebi_submission.study.ebi_submission_status = ( - "failed: ASCP - %s" % error) - LogEntry.create('Fatal', error, + "failed: ASCP - %s" % stderr) + LogEntry.create('Fatal', stderr, info={ 'ebi_submission': preprocessed_data_id, 'fail': cmd}) diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index e795443c6..609f751a9 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -882,8 +882,7 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): demux_samples = set() with open_file(demux) as demux_fh: if not isinstance(demux_fh, File): - error_msg = ("There was an error parsing the demux file: " - "%s" % demux) + error_msg = ("'%s' doesn't look like a demux file" % demux) LogEntry.create('Runtime', error_msg) raise EBISubmissionError(error_msg) for s, i in to_per_sample_ascii(demux_fh, diff --git a/setup_acsp.sh b/setup_acsp.sh deleted file mode 100644 index c6a97a0f2..000000000 --- a/setup_acsp.sh +++ /dev/null @@ -1,4 +0,0 @@ -openssl aes-256-cbc -K $encrypted_e698cf0e691c_key -iv $encrypted_e698cf0e691c_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d -wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh -chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh -./ascp-install-3.5.4.102989-linux-64-qiita.sh From 30723c2eee7efae70f3adc5918fd0f98aee5fe76 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 10 Oct 2015 05:16:19 -0600 Subject: [PATCH 11/16] fixing build errors --- .travis.yml | 11 +++++++++-- qiita_ware/commands.py | 28 ++++++++++++++++------------ qiita_ware/ebi.py | 2 +- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4633b0344..e09fd8a1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,8 +38,15 @@ script: - qiita-env start_cluster qiita-general - qiita-env make --no-load-ontologies - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi - - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi - - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage + - if [ ${TEST_ADD_STUDIES} == "False" ]; then + qiita-test-install ; + if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then + nosetests -I qiita_ware/test/test_ebi.py --with-doctest --with-coverage + else + nosetests --with-doctest --with-coverage + fi + fi + - if [ ${TEST_ADD_STUDIES} == "False" ]; then ; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then flake8 qiita_* setup.py scripts/qiita scripts/qiita-env scripts/qiita-test-install ; fi diff --git a/qiita_ware/commands.py b/qiita_ware/commands.py index 9bc2b8d56..3eda92d7d 100644 --- a/qiita_ware/commands.py +++ b/qiita_ware/commands.py @@ -69,13 +69,13 @@ def submit_EBI(preprocessed_data_id, action, send): except Exception as e: stdout = '' stderr = str(e) + le = LogEntry.create( + 'Fatal', "Command: %s\nError: %s\n" % (cmd, str(e)), + info={'ebi_submission': preprocessed_data_id}) ebi_submission.study.ebi_submission_status = ( - "failed: ASCP - %s" % stderr) - LogEntry.create('Fatal', stderr, - info={ - 'ebi_submission': preprocessed_data_id, - 'fail': cmd}) - raise + "failed: ASCP submission, log id: %d" % le.id) + raise ComputeError("EBI Submission failed! Log id: " + "%d" % le.id) finally: open(ebi_submission.ascp_reply, 'a').write( 'stdout:\n%s\n\nstderr: %s' % (stdout, stderr)) @@ -93,12 +93,16 @@ def submit_EBI(preprocessed_data_id, action, send): "%d" % preprocessed_data_id)) try: xml_content, stderr, _ = system_call(xmls_cmds) - except Exception as e: - error = str(e) - LogEntry.create('Fatal', error, - info={'ebi_submission': preprocessed_data_id, - 'fail': cmd}) + xml_content = '' + stderr = str(e) + le = LogEntry.create('Fatal', + "Command: %s\nError: %s\n" % ( + cmd, str(e)), + info={'ebi_submission': preprocessed_data_id}) + ebi_submission.study.ebi_submission_status = ( + "failed: XML submission, log id: %d" % le.id) + raise ComputeError("EBI Submission failed! Log id: %d" % le.id) else: LogEntry.create('Runtime', ('Submission of sequences of pre_processed_id: ' @@ -116,7 +120,7 @@ def submit_EBI(preprocessed_data_id, action, send): 'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)), info={'ebi_submission': preprocessed_data_id}) ebi_submission.study.ebi_submission_status = ( - "failed: XML submission, log id: %d" % le.id) + "failed: XML parsing, log id: %d" % le.id) raise ComputeError("EBI Submission failed! Log id: %d" % le.id) ebi_submission.study.ebi_submission_status = 'submitted' diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index 609f751a9..031e3c5ab 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -882,7 +882,7 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): demux_samples = set() with open_file(demux) as demux_fh: if not isinstance(demux_fh, File): - error_msg = ("'%s' doesn't look like a demux file" % demux) + error_msg = "'%s' doesn't look like a demux file" % demux LogEntry.create('Runtime', error_msg) raise EBISubmissionError(error_msg) for s, i in to_per_sample_ascii(demux_fh, From f57a3640a68bb50793658a102fcd061997f850fb Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 10 Oct 2015 05:27:37 -0600 Subject: [PATCH 12/16] testing if --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e09fd8a1a..10a521cff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,10 +41,10 @@ script: - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then - nosetests -I qiita_ware/test/test_ebi.py --with-doctest --with-coverage + nosetests -I qiita_ware/test/test_ebi.py --with-doctest --with-coverage; else - nosetests --with-doctest --with-coverage - fi + nosetests --with-doctest --with-coverage; + fi; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then ; fi From a7f7dfe47d95b1217b305c22ec70cc49e0a60891 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 10 Oct 2015 05:35:30 -0600 Subject: [PATCH 13/16] fixing leftover if --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 10a521cff..e5a73d654 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,8 +46,6 @@ script: nosetests --with-doctest --with-coverage; fi; fi - - if [ ${TEST_ADD_STUDIES} == "False" ]; then - ; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then flake8 qiita_* setup.py scripts/qiita scripts/qiita-env scripts/qiita-test-install ; fi - ls -R /home/travis/miniconda3/envs/env_name/lib/python2.7/site-packages/qiita_pet/support_files/doc/ From 9377a3a8246fcdabdcdeb7c3221606885e206880 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 10 Oct 2015 05:45:01 -0600 Subject: [PATCH 14/16] invert if --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5a73d654..b882b0a6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,9 +41,9 @@ script: - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then - nosetests -I qiita_ware/test/test_ebi.py --with-doctest --with-coverage; - else nosetests --with-doctest --with-coverage; + else + nosetests -I qiita_ware/test/test_ebi.py --with-doctest --with-coverage; fi; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then flake8 qiita_* setup.py scripts/qiita From d2eae77e197d8bed674c69c1400b03b39dd8ed66 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 10 Oct 2015 06:49:11 -0600 Subject: [PATCH 15/16] fixing ignore --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b882b0a6b..5414e8fea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,7 @@ script: if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then nosetests --with-doctest --with-coverage; else - nosetests -I qiita_ware/test/test_ebi.py --with-doctest --with-coverage; + nosetests --with-doctest --with-coverage --ignore-files='(^\.|^_|^test_ebi.py$)'; fi; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then flake8 qiita_* setup.py scripts/qiita From a368f419c8b4003dbeadf453086089b891219281 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 10 Oct 2015 08:20:52 -0600 Subject: [PATCH 16/16] fixing error --- .travis.yml | 10 ++----- qiita_ware/test/test_metadata_pipeline.py | 35 ++++++++++++----------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5414e8fea..ec24e9cf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,14 +38,8 @@ script: - qiita-env start_cluster qiita-general - qiita-env make --no-load-ontologies - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi - - if [ ${TEST_ADD_STUDIES} == "False" ]; then - qiita-test-install ; - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then - nosetests --with-doctest --with-coverage; - else - nosetests --with-doctest --with-coverage --ignore-files='(^\.|^_|^test_ebi.py$)'; - fi; - fi + - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi + - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then flake8 qiita_* setup.py scripts/qiita scripts/qiita-env scripts/qiita-test-install ; fi - ls -R /home/travis/miniconda3/envs/env_name/lib/python2.7/site-packages/qiita_pet/support_files/doc/ diff --git a/qiita_ware/test/test_metadata_pipeline.py b/qiita_ware/test/test_metadata_pipeline.py index 2cab14696..7a8bbc850 100644 --- a/qiita_ware/test/test_metadata_pipeline.py +++ b/qiita_ware/test/test_metadata_pipeline.py @@ -68,7 +68,7 @@ def test_create_templates_from_qiime_mapping_file(self): self.assertEqual(set(obs_st.categories()), exp) exp = {"barcode", "primer", "center_name", "run_prefix", "platform", - "library_construction_protocol", + "library_construction_protocol", "instrument_model", "experiment_design_description"} self.assertEqual(set(obs_pt.categories()), exp) @@ -94,7 +94,7 @@ def test_create_templates_from_qiime_mapping_file_reverse_linker(self): self.assertEqual(set(obs_st.categories()), exp) exp = {"barcode", "primer", "center_name", "run_prefix", "platform", - "library_construction_protocol", + "library_construction_protocol", "instrument_model", "experiment_design_description", "reverselinkerprimer"} self.assertEqual(set(obs_pt.categories()), exp) @@ -105,38 +105,41 @@ def test_create_templates_from_qiime_mapping_file_error(self): QIIME_MAP = ( - "#SampleID\tBarcodeSequence\tLinkerPrimerSequence\t" + "#SampleID\tBarcodeSequence\tLinkerPrimerSequence\tinstrument_model\t" "physical_specimen_location\tphysical_specimen_remaining\tdna_extracted\t" "sample_type\thost_subject_id\tlatitude\tlongitude\ttaxon_id\t" "scientific_name\tcenter_name\trun_prefix\tplatform\t" "library_construction_protocol\texperiment_design_description\t" "collection_timestamp\tDescription\n" - "Sample1\tGTCCGCAAGTTA\tGTGCCAGCMGCCGCGGTAA\tUCSD\tTRUE\tTRUE\ttype1\t" - "NotIdentified\t4.1\t4.1\t9606\thomo sapiens\tANL\trp_1\tILLUMINA\t" - "protocol_1\tedd_1\t05/28/15 11:00\tDescription S1\n" - "Sample2\tCGTAGAGCTCTC\tGTGCCAGCMGCCGCGGTAA\tUCSD\tTRUE\tTRUE\ttype2\t" - "NotIdentified\t4.2\t4.2\t9606\thomo sapiens\tANL\trp_1\tILLUMINA\t" - "protocol_1\tedd_1\t05/28/15 11:00\tDescription S2\n" - "Sample3\tCCTCTGAGAGCT\tGTGCCAGCMGCCGCGGTAA\tUCSD\tTRUE\tTRUE\ttype3\t" - "NotIdentified\t4.3\t4.3\t9606\thomo sapiens\tANL\trp_2\tILLUMINA\t" - "protocol_1\tedd_1\t05/28/15 11:00\tDescription S3\n") + "Sample1\tGTCCGCAAGTTA\tGTGCCAGCMGCCGCGGTAA\tIllumina MiSeq\tUCSD\tTRUE\t" + "TRUE\ttype1\tNotIdentified\t4.1\t4.1\t9606\thomo sapiens\tANL\trp_1\t" + "ILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S1\n" + "Sample2\tCGTAGAGCTCTC\tGTGCCAGCMGCCGCGGTAA\tIllumina MiSeq\tUCSD\tTRUE\t" + "TRUE\ttype2\tNotIdentified\t4.2\t4.2\t9606\thomo sapiens\tANL\trp_1\t" + "ILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S2\n" + "Sample3\tCCTCTGAGAGCT\tGTGCCAGCMGCCGCGGTAA\tIllumina MiSeq\tUCSD\tTRUE\t" + "TRUE\ttype3\tNotIdentified\t4.3\t4.3\t9606\thomo sapiens\tANL\trp_2\t" + "ILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S3\n") QIIME_MAP_WITH_REVERSE_LINKER_PRIMER = ( "#SampleID\tBarcodeSequence\tLinkerPrimerSequence\tReverseLinkerPrimer\t" "physical_specimen_location\tphysical_specimen_remaining\tdna_extracted\t" "sample_type\thost_subject_id\tlatitude\tlongitude\ttaxon_id\t" - "scientific_name\tcenter_name\trun_prefix\tplatform\t" + "scientific_name\tcenter_name\trun_prefix\tplatform\tinstrument_model\t" "library_construction_protocol\texperiment_design_description\t" "collection_timestamp\tDescription\n" "Sample1\tGTCCGCAAGTTA\tGTGCCAGCMGCCGCGGTAA\tGTGCCAGCMGCCGCGGTAA\tUCSD\t" "TRUE\tTRUE\ttype1\tNotIdentified\t4.1\t4.1\t9606\thomo sapiens\tANL\t" - "rp_1\tILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S1\n" + "rp_1\tILLUMINA\tIllumina MiSeq\tprotocol_1\tedd_1\t05/28/15 11:00\t" + "Description S1\n" "Sample2\tCGTAGAGCTCTC\tGTGCCAGCMGCCGCGGTAA\tGTGCCAGCMGCCGCGGTAA\tUCSD\t" "TRUE\tTRUE\ttype2\tNotIdentified\t4.2\t4.2\t9606\thomo sapiens\tANL\t" - "rp_1\tILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S2\n" + "rp_1\tILLUMINA\tIllumina MiSeq\tprotocol_1\tedd_1\t05/28/15 11:00\t" + "Description S2\n" "Sample3\tCCTCTGAGAGCT\tGTGCCAGCMGCCGCGGTAA\tGTGCCAGCMGCCGCGGTAA\tUCSD\t" "TRUE\tTRUE\ttype3\tNotIdentified\t4.3\t4.3\t9606\thomo sapiens\tANL\t" - "rp_2\tILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S3\n") + "rp_2\tILLUMINA\tIllumina MiSeq\tprotocol_1\tedd_1\t05/28/15 11:00\t" + "Description S3\n") QIIME_MAP_ERROR = ( "#SampleID\tBarcodeSequence\tphysical_specimen_location\t"