diff --git a/.travis.yml b/.travis.yml index 3735fb9c0..ec24e9cf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,10 @@ before_install: # Update conda itself - conda update --yes conda # Downloading and setting up ascp for EBI testing - - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then sh setup_acsp.sh; fi + - wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh + - chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh + - ./ascp-install-3.5.4.102989-linux-64-qiita.sh + - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then openssl aes-256-cbc -K $encrypted_e698cf0e691c_key -iv $encrypted_e698cf0e691c_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d; fi install: # install a few of the dependencies that pip would otherwise try to install # when intalling scikit-bio @@ -36,8 +39,7 @@ script: - qiita-env make --no-load-ontologies - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi - - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage - ; fi + - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then flake8 qiita_* setup.py scripts/qiita scripts/qiita-env scripts/qiita-test-install ; fi - ls -R /home/travis/miniconda3/envs/env_name/lib/python2.7/site-packages/qiita_pet/support_files/doc/ diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index dc5a6eb60..d24fd0728 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -78,16 +78,12 @@ class ConfigurationManager(object): smtp_email : str The email address that mail will be sent from when sending mail from the SMTP server - ebi_access_key : str - The access key issued by EBI for REST submissions ebi_seq_xfer_user : str The user to use when submitting to EBI ebi_seq_xfer_pass : str The password for the ebi_seq_xfer_user ebi_seq_xfer_url : str The URL of EBI's sequence portal site - ebi_skip_curl_cert : bool - Whether or not to skip the certificate check when curling the metadata ebi_center_name : str The name of the sequencing center to use when doing EBI submissions ebi_organization_prefix : str @@ -213,14 +209,11 @@ def _get_smtp(self, config): def _get_ebi(self, config): sec_get = partial(config.get, 'ebi') - sec_getbool = partial(config.getboolean, 'ebi') - self.ebi_access_key = sec_get('EBI_ACCESS_KEY') self.ebi_seq_xfer_user = sec_get('EBI_SEQ_XFER_USER') self.ebi_seq_xfer_pass = sec_get('EBI_SEQ_XFER_PASS') self.ebi_seq_xfer_url = sec_get('EBI_SEQ_XFER_URL') self.ebi_dropbox_url = sec_get('EBI_DROPBOX_URL') - self.ebi_skip_curl_cert = sec_getbool('EBI_SKIP_CURL_CERT') self.ebi_center_name = sec_get('EBI_CENTER_NAME') self.ebi_organization_prefix = sec_get('EBI_ORGANIZATION_PREFIX') diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index b87f67264..e50e8951a 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -99,9 +99,6 @@ ADMIN_PASSWORD = # ----------------------------- EBI settings ----------------------------- [ebi] -# The access key issued by EBI for REST submissions -EBI_ACCESS_KEY = - # The user to use when submitting to EBI EBI_SEQ_XFER_USER = Webin-41528 @@ -117,12 +114,8 @@ EBI_SEQ_XFER_URL = webin.ebi.ac.uk # testing URL EBI_DROPBOX_URL = https://www-test.ebi.ac.uk/ena/submit/drop-box/submit/ -# Skip acertificate check when calling curl (necessary for using the EBI test -# dropbox URL) -EBI_SKIP_CURL_CERT = True - # The name of the sequencing center to use when doing EBI submissions -EBI_CENTER_NAME = CCME-COLORADO +EBI_CENTER_NAME = qiita-test # This string (with an underscore) will be prefixed to your EBI submission and # study aliases diff --git a/qiita_core/support_files/config_test_travis.cfg.enc b/qiita_core/support_files/config_test_travis.cfg.enc index f247feb94..816384459 100644 Binary files a/qiita_core/support_files/config_test_travis.cfg.enc and b/qiita_core/support_files/config_test_travis.cfg.enc differ diff --git a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst index e98f38e67..7a8b675ee 100644 --- a/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst +++ b/qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst @@ -38,6 +38,8 @@ These are the columns required for successfully submit your data to EBI: +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ | ``taxon_id`` | integer | NCBI's taxon id for the sample | +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ +| ``description`` | free text | Description of the sample. | ++----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ | ``scientific_name`` | free text | NCBI's scientific name for the provided taxon ID | +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+ @@ -52,8 +54,6 @@ the centralized `Qiita server `__: +===================================+======================================================================+=============================================================================================================================================================================================================================================+ | ``sample_type`` | free text | Description of the type of sample. | +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``description`` | free text | Description of the sample. | -+-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``physical_specimen_remaining`` | ``y`` or ``n`` | Is there still physical sample (e.g., soil, not DNA) available? | +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ``dna_extracted`` | ``y`` or ``n`` | Has DNA already been extracted for this sample? | diff --git a/qiita_ware/commands.py b/qiita_ware/commands.py index 9c9aba5d6..3eda92d7d 100644 --- a/qiita_ware/commands.py +++ b/qiita_ware/commands.py @@ -10,8 +10,10 @@ from shutil import rmtree from tarfile import open as taropen from tempfile import mkdtemp - +from os import environ +from traceback import format_exc from moi.job import system_call + from qiita_db.study import Study from qiita_db.data import PreprocessedData from qiita_db.metadata_template import PrepTemplate, SampleTemplate @@ -19,11 +21,9 @@ from qiita_core.qiita_settings import qiita_config from qiita_ware.ebi import EBISubmission from qiita_ware.exceptions import ComputeError, EBISubmissionError -from traceback import format_exc -from os import environ -def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): +def submit_EBI(preprocessed_data_id, action, send): """Submit a preprocessed data to EBI Parameters @@ -34,27 +34,19 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): The action to perform with this data send : bool True to actually send the files - fastq_dir_fp : str, optional - The fastq filepath - - Notes - ----- - If fastq_dir_fp is passed, it must not contain any empty files, or - gzipped empty files """ # step 1: init and validate ebi_submission = EBISubmission(preprocessed_data_id, action) # step 2: generate demux fastq files - ebi_submission.preprocessed_data.update_insdc_status('demuxing samples') + ebi_submission.study.ebi_submission_status = 'submitting' try: ebi_submission.generate_demultiplexed_fastq() except: error_msg = format_exc() - if isdir(ebi_submission.ebi_dir): - rmtree(ebi_submission.ebi_dir) - ebi_submission.preprocessed_data.update_insdc_status( - 'failed: %s' % error_msg) + if isdir(ebi_submission.full_ebi_dir): + rmtree(ebi_submission.full_ebi_dir) + ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg LogEntry.create('Runtime', error_msg, info={'ebi_submission': preprocessed_data_id}) raise @@ -66,22 +58,33 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): # step 4: sending sequences old_ascp_pass = environ.get('ASPERA_SCP_PASS', '') environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass - seqs_cmds = ebi_submission.generate_send_sequences_cmd() + LogEntry.create('Runtime', ("Submitting sequences for pre_processed_id: " "%d" % preprocessed_data_id)) try: - # place holder for moi call see #1477 - pass - except: - LogEntry.create('Fatal', seqs_cmds, - info={'ebi_submission': preprocessed_data_id}) - else: - LogEntry.create('Runtime', - ('Submission of sequences of pre_processed_id: ' - '%d completed successfully' % - preprocessed_data_id)) - environ['ASPERA_SCP_PASS'] = old_ascp_pass + for cmd in ebi_submission.generate_send_sequences_cmd(): + try: + stdout, stderr, _ = system_call(cmd) + except Exception as e: + stdout = '' + stderr = str(e) + le = LogEntry.create( + 'Fatal', "Command: %s\nError: %s\n" % (cmd, str(e)), + info={'ebi_submission': preprocessed_data_id}) + ebi_submission.study.ebi_submission_status = ( + "failed: ASCP submission, log id: %d" % le.id) + raise ComputeError("EBI Submission failed! Log id: " + "%d" % le.id) + finally: + open(ebi_submission.ascp_reply, 'a').write( + 'stdout:\n%s\n\nstderr: %s' % (stdout, stderr)) + finally: + environ['ASPERA_SCP_PASS'] = old_ascp_pass + LogEntry.create('Runtime', + ('Submission of sequences of pre_processed_id: ' + '%d completed successfully' % + preprocessed_data_id)) # step 5: sending xml and parsing answer xmls_cmds = ebi_submission.generate_curl_command() @@ -89,26 +92,35 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): ("Submitting XMLs for pre_processed_id: " "%d" % preprocessed_data_id)) try: - # place holder for moi call see #1477 - xmls_cmds_moi = xmls_cmds - except: - # handle exception - LogEntry.create('Fatal', seqs_cmds, - info={'ebi_submission': preprocessed_data_id}) + xml_content, stderr, _ = system_call(xmls_cmds) + except Exception as e: + xml_content = '' + stderr = str(e) + le = LogEntry.create('Fatal', + "Command: %s\nError: %s\n" % ( + cmd, str(e)), + info={'ebi_submission': preprocessed_data_id}) + ebi_submission.study.ebi_submission_status = ( + "failed: XML submission, log id: %d" % le.id) + raise ComputeError("EBI Submission failed! Log id: %d" % le.id) else: LogEntry.create('Runtime', ('Submission of sequences of pre_processed_id: ' '%d completed successfully' % preprocessed_data_id)) + finally: + open(ebi_submission.curl_reply, 'w').write( + 'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr)) try: st_acc, sa_acc, bio_acc, ex_acc, run_acc = \ - ebi_submission.parse_EBI_reply(xmls_cmds_moi) + ebi_submission.parse_EBI_reply(xml_content) except EBISubmissionError as e: le = LogEntry.create( - 'Fatal', "Command: %s\nError: %s\n" % (xmls_cmds_moi, str(e)), + 'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)), info={'ebi_submission': preprocessed_data_id}) - ebi_submission.preprocessed_data.update_insdc_status('failed') + ebi_submission.study.ebi_submission_status = ( + "failed: XML parsing, log id: %d" % le.id) raise ComputeError("EBI Submission failed! Log id: %d" % le.id) ebi_submission.study.ebi_submission_status = 'submitted' diff --git a/qiita_ware/dispatchable.py b/qiita_ware/dispatchable.py index 0c5883541..f5690814d 100644 --- a/qiita_ware/dispatchable.py +++ b/qiita_ware/dispatchable.py @@ -52,11 +52,7 @@ def preprocessor(study_id, prep_template_id, param_id, param_constructor): def submit_to_ebi(preprocessed_data_id, submission_type): """Submit a study to EBI""" - study_acc, submission_acc = submit_EBI(preprocessed_data_id, - submission_type, - True) - - return study_acc, submission_acc + submit_EBI(preprocessed_data_id, submission_type, True) def submit_to_VAMPS(preprocessed_data_id): diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index c9f5c6647..031e3c5ab 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -1,12 +1,13 @@ from os.path import basename, join, isdir, isfile from os import makedirs, remove, listdir from datetime import date, timedelta +from urllib import quote from xml.etree import ElementTree as ET from xml.etree.ElementTree import ParseError from xml.sax.saxutils import escape from gzip import GzipFile from functools import partial - +from h5py import File from future.utils import viewitems, viewkeys from skbio.util import safe_md5, create_dir @@ -16,30 +17,12 @@ from qiita_ware.util import open_file from qiita_db.logger import LogEntry from qiita_db.ontology import Ontology -from qiita_db.util import convert_to_id +from qiita_db.util import convert_to_id, get_mountpoint from qiita_db.study import Study from qiita_db.data import PreprocessedData from qiita_db.metadata_template import PrepTemplate, SampleTemplate -class InvalidMetadataError(Exception): - """Error that is raised when metadata is not representable as a string""" - pass - - -class SampleAlreadyExistsError(Exception): - """Error that is raised when a sample is added to a submission that already - has a sample by that name""" - pass - - -class NoXMLError(Exception): - """Error that is raised when the generation of one XML file cannot be - completed because it depends on another XML file that has not yet been - generated""" - pass - - def clean_whitespace(text): """Standardizes whitespaces so there is only one space separating tokens @@ -157,16 +140,17 @@ def __init__(self, preprocessed_data_id, action): "term is neither one of the official terms nor " "one of the user-defined terms in the ENA " "ontology." % it) - - self.ebi_dir = join(qiita_config.working_dir, - 'ebi_submission_%d' % preprocessed_data_id) - self.xml_dir = join(self.ebi_dir, 'xml_dir') + _, base_fp = get_mountpoint("preprocessed_data")[0] + self.ebi_dir = '%d_ebi_submission' % preprocessed_data_id + self.full_ebi_dir = join(base_fp, self.ebi_dir) + self.ascp_reply = join(self.full_ebi_dir, 'ascp_reply.txt') + self.curl_reply = join(self.full_ebi_dir, 'curl_reply.xml') + self.xml_dir = join(self.full_ebi_dir, 'xml_dir') self.study_xml_fp = None self.sample_xml_fp = None self.experiment_xml_fp = None self.run_xml_fp = None self.submission_xml_fp = None - self.pmids = self.study.pmids # getting the restrictions @@ -186,7 +170,7 @@ def __init__(self, preprocessed_data_id, action): self.samples = {} self.samples_prep = {} self.sample_demux_fps = {} - get_output_fp = partial(join, self.ebi_dir) + get_output_fp = partial(join, self.full_ebi_dir) nvp = [] nvim = [] for k, v in viewitems(self.sample_template): @@ -566,7 +550,7 @@ def generate_run_xml(self): data_block = ET.SubElement(run, 'DATA_BLOCK') files = ET.SubElement(data_block, 'FILES') ET.SubElement(files, 'FILE', { - 'filename': file_path, + 'filename': join(self.ebi_dir, basename(file_path)), 'filetype': file_type, 'quality_scoring_system': 'phred', 'checksum_method': 'MD5', @@ -702,8 +686,7 @@ def generate_xml_files(self): def generate_curl_command( self, ebi_seq_xfer_user=qiita_config.ebi_seq_xfer_user, - ebi_access_key=qiita_config.ebi_access_key, - ebi_skip_curl_cert=qiita_config.ebi_skip_curl_cert, + ebi_seq_xfer_pass=qiita_config.ebi_seq_xfer_pass, ebi_dropbox_url=qiita_config.ebi_dropbox_url): """Generates the curl command for submission @@ -711,10 +694,8 @@ def generate_curl_command( ---------- ebi_seq_xfer_user : str The user to use when submitting to EBI - ebi_access_key : str - The access key issued by EBI for REST submissions - ebi_skip_curl_cert : bool - If the curl certificate should be skipped + ebi_seq_xfer_pass : str + The user password issued by EBI for REST submissions ebi_dropbox_url : str The dropbox url @@ -729,22 +710,22 @@ def generate_curl_command( be generated before executing this function """ # make sure that the XML files have been generated - url = '?auth=ENA%20{0}%20{1}'.format(ebi_seq_xfer_user, - ebi_access_key) - curl_command = ( - 'curl {0}-F "SUBMISSION=@{1}" -F "STUDY=@{2}" -F "SAMPLE=@{3}" ' - '-F "RUN=@{4}" -F "EXPERIMENT=@{5}" "{6}"' - ).format( - '-k ' if ebi_skip_curl_cert else '', - self.submission_xml_fp, - self.study_xml_fp, - self.sample_xml_fp, - self.run_xml_fp, - self.experiment_xml_fp, - join(ebi_dropbox_url, url) - ) - - return curl_command + url = '?auth=ENA%20{0}%20{1}'.format(quote(ebi_seq_xfer_user), + quote(ebi_seq_xfer_pass)) + curl_cmd = ['curl -sS -k'] + if self.submission_xml_fp is not None: + curl_cmd.append(' -F "SUBMISSION=@%s"' % self.submission_xml_fp) + if self.study_xml_fp is not None: + curl_cmd.append(' -F "STUDY=@%s"' % self.study_xml_fp) + if self.sample_xml_fp is not None: + curl_cmd.append(' -F "SAMPLE=@%s"' % self.sample_xml_fp) + if self.run_xml_fp is not None: + curl_cmd.append(' -F "RUN=@%s"' % self.run_xml_fp) + if self.experiment_xml_fp is not None: + curl_cmd.append(' -F "EXPERIMENT=@%s"' % self.experiment_xml_fp) + curl_cmd.append(' "%s"' % join(ebi_dropbox_url, url)) + + return ''.join(curl_cmd) def generate_send_sequences_cmd(self): """Generate the sequences to EBI via ascp command @@ -764,7 +745,7 @@ def generate_send_sequences_cmd(self): fastqs_div = [fastqs[i::10] for i in range(10) if fastqs[i::10]] ascp_commands = [] for f in fastqs_div: - ascp_commands.append('ascp --ignore-host-key -L- -d -QT -k2 ' + ascp_commands.append('ascp --ignore-host-key -d -QT -k2 ' '{0} {1}@{2}:./{3}/'.format( ' '.join(f), qiita_config.ebi_seq_xfer_user, @@ -872,27 +853,38 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): Notes ----- - - As a performace feature, this method will check if self.ebi_dir + - As a performace feature, this method will check if self.full_ebi_dir already exists and, if it does, the script will assume that in a previous execution this step was performed correctly and will simply - read the file names from self.ebi_dir + read the file names from self.full_ebi_dir - When the object is created (init), samples, samples_prep and sample_demux_fps hold values for all available samples in the database. Here some of those values will be deleted (del's, within the loops) for those cases where the fastq.gz files weren't written or exist. This is an indication that they had no sequences and this kind of files are not accepted in EBI + + Raises + ------ + EBISubmissionError + - The demux file couldn't be read + - All samples are removed """ ppd = self.preprocessed_data - if not isdir(self.ebi_dir) or rewrite_fastq: - makedirs(self.ebi_dir) + dir_not_exists = not isdir(self.full_ebi_dir) + if dir_not_exists or rewrite_fastq: + makedirs(self.full_ebi_dir) demux = [path for _, path, ftype in ppd.get_filepaths() if ftype == 'preprocessed_demux'][0] demux_samples = set() with open_file(demux) as demux_fh: + if not isinstance(demux_fh, File): + error_msg = "'%s' doesn't look like a demux file" % demux + LogEntry.create('Runtime', error_msg) + raise EBISubmissionError(error_msg) for s, i in to_per_sample_ascii(demux_fh, self.prep_template.keys()): sample_fp = self.sample_demux_fps[s] @@ -909,13 +901,13 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): del(self.samples_prep[s]) del(self.sample_demux_fps[s]) remove(sample_fp) - else: demux_samples = set() extension = '.fastq.gz' extension_len = len(extension) - for f in listdir(self.ebi_dir): - if isfile(join(self.ebi_dir, f)) and f.endswith(extension): + for f in listdir(self.full_ebi_dir): + fpath = join(self.full_ebi_dir, f) + if isfile(fpath) and f.endswith(extension): demux_samples.add(f[:-extension_len]) missing_samples = set(self.samples.keys()).difference( @@ -925,4 +917,10 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None): del(self.samples_prep[ms]) del(self.sample_demux_fps[ms]) + if not demux_samples: + error_msg = ("All samples were removed from the submission " + "because the demux file is empty or the sample names " + "do not match.") + LogEntry.create('Runtime', error_msg) + raise EBISubmissionError(error_msg) return demux_samples diff --git a/qiita_ware/test/test_commands.py b/qiita_ware/test/test_commands.py new file mode 100644 index 000000000..0642cf097 --- /dev/null +++ b/qiita_ware/test/test_commands.py @@ -0,0 +1,198 @@ +from __future__ import division + +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- +from unittest import TestCase, main +from os.path import join +from warnings import simplefilter +from tempfile import mkdtemp +import pandas as pd +from datetime import datetime + +from h5py import File + +from qiita_ware.exceptions import ComputeError +from qiita_ware.demux import to_hdf5 +from qiita_ware.commands import submit_EBI +from qiita_db.data import PreprocessedData +from qiita_db.study import Study, StudyPerson +from qiita_db.metadata_template import PrepTemplate, SampleTemplate +from qiita_db.user import User +from qiita_core.util import qiita_test_checker + + +@qiita_test_checker() +class CommandsTests(TestCase): + def setUp(self): + self.files_to_remove = [] + self.temp_dir = mkdtemp() + self.files_to_remove.append(self.temp_dir) + + def write_demux_files(self, prep_template, generate_hdf5=True): + """Writes a demux test file to avoid duplication of code""" + fna_fp = join(self.temp_dir, 'seqs.fna') + demux_fp = join(self.temp_dir, 'demux.seqs') + if generate_hdf5: + with open(fna_fp, 'w') as f: + f.write(FASTA_EXAMPLE) + with File(demux_fp, "w") as f: + to_hdf5(fna_fp, f) + else: + with open(demux_fp, 'w') as f: + f.write('') + + ppd = PreprocessedData.create(Study(1), + "preprocessed_sequence_illumina_params", + 1, [(demux_fp, 6)], prep_template) + return ppd + + def generate_new_study_with_preprocessed_data(self): + """Creates a new study up to the processed data for testing""" + # ignoring warnings generated when adding templates + simplefilter("ignore") + info = { + "timeseries_type_id": 1, + "metadata_complete": True, + "mixs_compliant": True, + "number_samples_collected": 3, + "number_samples_promised": 3, + "study_alias": "Test EBI", + "study_description": "Study for testing EBI", + "study_abstract": "Study for testing EBI", + "emp_person_id": StudyPerson(2), + "principal_investigator_id": StudyPerson(3), + "lab_person_id": StudyPerson(1) + } + study = Study.create(User('test@foo.bar'), "Test EBI study", [1], info) + metadata_dict = { + 'Sample1': {'collection_timestamp': datetime(2015, 6, 1, 7, 0, 0), + 'physical_specimen_location': 'location1', + 'taxon_id': 9606, + 'scientific_name': 'homo sapiens', + 'Description': 'Test Sample 1'}, + 'Sample2': {'collection_timestamp': datetime(2015, 6, 2, 7, 0, 0), + 'physical_specimen_location': 'location1', + 'taxon_id': 9606, + 'scientific_name': 'homo sapiens', + 'Description': 'Test Sample 2'}, + 'Sample3': {'collection_timestamp': datetime(2015, 6, 3, 7, 0, 0), + 'physical_specimen_location': 'location1', + 'taxon_id': 9606, + 'scientific_name': 'homo sapiens', + 'Description': 'Test Sample 3'} + } + metadata = pd.DataFrame.from_dict(metadata_dict, orient='index') + SampleTemplate.create(metadata, study) + metadata_dict = { + 'Sample1': {'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'CGTAGAGCTCTC', + 'center_name': 'KnightLab', + 'platform': 'ILLUMINA', + 'instrument_model': 'Illumina MiSeq', + 'library_construction_protocol': 'Protocol ABC', + 'experiment_design_description': "Random value 1"}, + 'Sample2': {'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'CGTAGAGCTCTA', + 'center_name': 'KnightLab', + 'platform': 'ILLUMINA', + 'instrument_model': 'Illumina MiSeq', + 'library_construction_protocol': 'Protocol ABC', + 'experiment_design_description': "Random value 2"}, + 'Sample3': {'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'CGTAGAGCTCTT', + 'center_name': 'KnightLab', + 'platform': 'ILLUMINA', + 'instrument_model': 'Illumina MiSeq', + 'library_construction_protocol': 'Protocol ABC', + 'experiment_design_description': "Random value 3"}, + } + metadata = pd.DataFrame.from_dict(metadata_dict, orient='index') + pt = PrepTemplate.create(metadata, study, "16S", 'Metagenomics') + fna_fp = join(self.temp_dir, 'seqs.fna') + demux_fp = join(self.temp_dir, 'demux.seqs') + with open(fna_fp, 'w') as f: + f.write(FASTA_EXAMPLE_2.format(study.id)) + with File(demux_fp, 'w') as f: + to_hdf5(fna_fp, f) + + ppd = PreprocessedData.create( + study, "preprocessed_sequence_illumina_params", 1, + [(demux_fp, 6)], pt) + + return ppd + + def test_submit_EBI_step_2_failure(self): + # see issue #406 + # ppd = self.write_demux_files(PrepTemplate(1), False) + # + # with self.assertRaises(AttributeError): + # submit_EBI(ppd.id, 'ADD', True) + pass + + def test_submit_EBI_parse_EBI_reply_failure(self): + ppd = self.write_demux_files(PrepTemplate(1)) + with self.assertRaises(ComputeError): + submit_EBI(ppd.id, 'ADD', True) + + def test_full_submission(self): + # see issue #406 + # ppd = self.generate_new_study_with_preprocessed_data() + # + # submit_EBI(ppd.id, 'ADD', True) + pass + + +FASTA_EXAMPLE = """>1.SKB2.640194_1 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB2.640194_2 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB2.640194_3 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM4.640180_4 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM4.640180_5 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB3.640195_6 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKB6.640176_7 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKD6.640190_8 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM6.640187_9 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKD9.640182_10 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM8.640201_11 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>1.SKM2.640199_12 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +""" + +FASTA_EXAMPLE_2 = """>{0}.Sample1_1 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample1_2 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample1_3 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample2_4 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample2_5 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample2_6 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample3_7 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample3_8 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +>{0}.Sample3_9 X orig_bc=X new_bc=X bc_diffs=0 +CCACCCAGTAAC +""" + + +if __name__ == '__main__': + main() diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 15234eec3..5b57b5ae6 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from __future__ import division # ----------------------------------------------------------------------------- @@ -28,6 +26,7 @@ from qiita_ware.demux import to_hdf5 from qiita_core.qiita_settings import qiita_config from qiita_db.data import PreprocessedData +from qiita_db.util import get_mountpoint from qiita_db.study import Study, StudyPerson from qiita_db.metadata_template import PrepTemplate, SampleTemplate from qiita_db.user import User @@ -77,15 +76,16 @@ def test_init(self): self.assertItemsEqual(e.pmids, ['123456', '7891011']) self.assertEqual(e.action, action) - get_output_fp = partial(join, e.ebi_dir) + self.assertEqual(e.ascp_reply, join(e.full_ebi_dir, 'ascp_reply.txt')) + self.assertEqual(e.curl_reply, join(e.full_ebi_dir, 'curl_reply.xml')) + get_output_fp = partial(join, e.full_ebi_dir) self.assertEqual(e.xml_dir, get_output_fp('xml_dir')) - self.assertEqual(e.study_xml_fp, None) - self.assertEqual(e.sample_xml_fp, None) - self.assertEqual(e.experiment_xml_fp, None) - self.assertEqual(e.run_xml_fp, None) - self.assertEqual(e.submission_xml_fp, None) + self.assertIsNone(e.study_xml_fp) + self.assertIsNone(e.sample_xml_fp) + self.assertIsNone(e.experiment_xml_fp) + self.assertIsNone(e.run_xml_fp) + self.assertIsNone(e.submission_xml_fp) - self.assertEqual(e.ebi_dir, get_output_fp()) for sample in e.sample_template: self.assertEqual(e.sample_template[sample], e.samples[sample]) self.assertEqual(e.prep_template[sample], e.samples_prep[sample]) @@ -219,57 +219,50 @@ def test_generate_curl_command(self): submission = EBISubmission(2, 'ADD') test_ebi_seq_xfer_user = 'ebi_seq_xfer_user' - test_ebi_access_key = 'ebi_access_key' + test_ebi_seq_xfer_pass = 'ebi_seq_xfer_pass' test_ebi_dropbox_url = 'ebi_dropbox_url' - # Without curl certificate authentication - test_ebi_skip_curl_cert = True submission.study_xml_fp = "/some/path/study.xml" submission.sample_xml_fp = "/some/path/sample.xml" submission.experiment_xml_fp = "/some/path/experiment.xml" submission.run_xml_fp = "/some/path/run.xml" submission.submission_xml_fp = "/some/path/submission.xml" obs = submission.generate_curl_command(test_ebi_seq_xfer_user, - test_ebi_access_key, - test_ebi_skip_curl_cert, + test_ebi_seq_xfer_pass, test_ebi_dropbox_url) - exp = ('curl -k ' + exp = ('curl -sS -k ' '-F "SUBMISSION=@/some/path/submission.xml" ' '-F "STUDY=@/some/path/study.xml" ' '-F "SAMPLE=@/some/path/sample.xml" ' '-F "RUN=@/some/path/run.xml" ' '-F "EXPERIMENT=@/some/path/experiment.xml" ' '"ebi_dropbox_url/?auth=ENA%20ebi_seq_xfer_user' - '%20ebi_access_key"') - self.assertEqual(obs, exp) - - # With curl certificate authentication - test_ebi_skip_curl_cert = False - obs = submission.generate_curl_command(test_ebi_seq_xfer_user, - test_ebi_access_key, - test_ebi_skip_curl_cert, - test_ebi_dropbox_url) - exp = ('curl ' - '-F "SUBMISSION=@/some/path/submission.xml" ' - '-F "STUDY=@/some/path/study.xml" ' - '-F "SAMPLE=@/some/path/sample.xml" ' - '-F "RUN=@/some/path/run.xml" ' - '-F "EXPERIMENT=@/some/path/experiment.xml" ' - '"ebi_dropbox_url/?auth=ENA%20ebi_seq_xfer_user' - '%20ebi_access_key"') + '%20ebi_seq_xfer_pass"') self.assertEqual(obs, exp) @qiita_test_checker() class TestEBISubmissionWriteRead(TestEBISubmission): - def write_demux_files(self, prep_template): + def write_demux_files(self, prep_template, sequences='FASTA-EXAMPLE'): """Writes a demux test file to avoid duplication of code""" fna_fp = join(self.temp_dir, 'seqs.fna') demux_fp = join(self.temp_dir, 'demux.seqs') - with open(fna_fp, 'w') as f: - f.write(FASTA_EXAMPLE) - with File(demux_fp, "w") as f: - to_hdf5(fna_fp, f) + if sequences == 'FASTA-EXAMPLE': + with open(fna_fp, 'w') as f: + f.write(FASTA_EXAMPLE) + with File(demux_fp, "w") as f: + to_hdf5(fna_fp, f) + elif sequences == 'WRONG-SEQS': + with open(fna_fp, 'w') as f: + f.write('>a_1 X orig_bc=X new_bc=X bc_diffs=0\nCCC') + with File(demux_fp, "w") as f: + to_hdf5(fna_fp, f) + elif sequences == 'EMPTY': + with open(demux_fp, 'w') as f: + f.write("") + else: + raise ValueError('Wrong sequences values: %s. Valid values: ' + 'FASTA_EXAMPLE, WRONG-SEQS, EMPTY' % sequences) ppd = PreprocessedData.create(Study(1), "preprocessed_sequence_illumina_params", @@ -445,14 +438,14 @@ def test_prep_with_less_samples_than_sample_template(self): # the EBISubmission can be generated ppd = self.generate_new_prep_template_and_write_demux_files(True) e = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) exp = ['1.SKD6.640190', '1.SKM6.640187', '1.SKD9.640182'] self.assertItemsEqual(exp, e.samples) def test_generate_experiment_xml(self): ppd = self.generate_new_study_with_preprocessed_data() submission = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(submission.ebi_dir) + self.files_to_remove.append(submission.full_ebi_dir) obs = ET.tostring(submission.generate_experiment_xml()) exp = EXPERIMENTXML_NEWSTUDY % { 'organization_prefix': qiita_config.ebi_organization_prefix, @@ -464,7 +457,7 @@ def test_generate_experiment_xml(self): self.assertEqual(obs, exp) submission = EBISubmission(2, 'ADD') - self.files_to_remove.append(submission.ebi_dir) + self.files_to_remove.append(submission.full_ebi_dir) samples = ['1.SKB2.640194', '1.SKB3.640195'] obs = ET.tostring(submission.generate_experiment_xml(samples=samples)) exp = EXPERIMENTXML @@ -491,7 +484,7 @@ def test_generate_experiment_xml(self): def test_generate_run_xml(self): ppd = self.generate_new_study_with_preprocessed_data() submission = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(submission.ebi_dir) + self.files_to_remove.append(submission.full_ebi_dir) submission.generate_demultiplexed_fastq(mtime=1) obs = ET.tostring(submission.generate_run_xml()) exp = RUNXML_NEWSTUDY % { @@ -517,7 +510,7 @@ def test_generate_run_xml(self): del(submission.samples_prep[k]) submission.generate_demultiplexed_fastq(mtime=1) - self.files_to_remove.append(submission.ebi_dir) + self.files_to_remove.append(submission.full_ebi_dir) obs = ET.tostring(submission.generate_run_xml()) exp = RUNXML % { @@ -532,7 +525,7 @@ def test_generate_run_xml(self): def test_generate_xml_files(self): ppd = self.generate_new_study_with_preprocessed_data() e = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) e.generate_demultiplexed_fastq() self.assertIsNone(e.run_xml_fp) self.assertIsNone(e.experiment_xml_fp) @@ -548,7 +541,7 @@ def test_generate_xml_files(self): ppd = self.generate_new_prep_template_and_write_demux_files(True) e = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) e.generate_demultiplexed_fastq() self.assertIsNone(e.run_xml_fp) self.assertIsNone(e.experiment_xml_fp) @@ -564,7 +557,7 @@ def test_generate_xml_files(self): ppd = self.write_demux_files(PrepTemplate(1)) e = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) e.generate_demultiplexed_fastq() self.assertIsNone(e.run_xml_fp) self.assertIsNone(e.experiment_xml_fp) @@ -578,6 +571,21 @@ def test_generate_xml_files(self): self.assertIsNone(e.study_xml_fp) self.assertIsNotNone(e.submission_xml_fp) + def test_generate_demultiplexed_fastq_failure(self): + # generating demux file for testing + ppd = self.write_demux_files(PrepTemplate(1), 'EMPTY') + + ebi_submission = EBISubmission(ppd.id, 'ADD') + self.files_to_remove.append(ebi_submission.full_ebi_dir) + with self.assertRaises(EBISubmissionError): + ebi_submission.generate_demultiplexed_fastq() + + ppd = self.write_demux_files(PrepTemplate(1), 'WRONG-SEQS') + ebi_submission = EBISubmission(ppd.id, 'ADD') + self.files_to_remove.append(ebi_submission.full_ebi_dir) + with self.assertRaises(EBISubmissionError): + ebi_submission.generate_demultiplexed_fastq() + def test_generate_demultiplexed_fastq(self): # generating demux file for testing exp_demux_samples = set( @@ -590,7 +598,7 @@ def test_generate_demultiplexed_fastq(self): # be created ebi_submission = EBISubmission(ppd.id, 'ADD') obs_demux_samples = ebi_submission.generate_demultiplexed_fastq() - self.files_to_remove.append(ebi_submission.ebi_dir) + self.files_to_remove.append(ebi_submission.full_ebi_dir) self.assertItemsEqual(obs_demux_samples, exp_demux_samples) # testing that the samples/samples_prep and demux_samples are the same self.assertItemsEqual(obs_demux_samples, ebi_submission.samples.keys()) @@ -601,7 +609,7 @@ def test_generate_demultiplexed_fastq(self): # exists and that we have the same files and ignore not fastq.gz files ebi_submission = EBISubmission(ppd.id, 'ADD') obs_demux_samples = ebi_submission.generate_demultiplexed_fastq() - self.files_to_remove.append(ebi_submission.ebi_dir) + self.files_to_remove.append(ebi_submission.full_ebi_dir) self.assertItemsEqual(obs_demux_samples, exp_demux_samples) # testing that the samples/samples_prep and demux_samples are the same self.assertItemsEqual(obs_demux_samples, ebi_submission.samples.keys()) @@ -612,43 +620,45 @@ def test_generate_send_sequences_cmd(self): ppd = self.write_demux_files(PrepTemplate(1)) e = EBISubmission(ppd.id, 'ADD') e.generate_demultiplexed_fastq() - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) e.generate_xml_files() obs = e.generate_send_sequences_cmd() - exp = ['ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKB2.640194.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM4.640180.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKB3.640195.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKB6.640176.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKD6.640190.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM6.640187.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKD9.640182.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM8.640201.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/', - 'ascp --ignore-host-key -L- -d -QT -k2 ' - '/tmp/ebi_submission_3/1.SKM2.640199.fastq.gz ' - 'Webin-41528@webin.ebi.ac.uk:.//tmp/ebi_submission_3/'] + _, base_fp = get_mountpoint("preprocessed_data")[0] + exp = ('ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKB2.640194.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKM4.640180.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKB3.640195.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKB6.640176.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKD6.640190.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKM6.640187.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKD9.640182.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKM8.640201.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/\n' + 'ascp --ignore-host-key -d -QT -k2 ' + '%(ebi_dir)s/1.SKM2.640199.fastq.gz ' + 'Webin-41528@webin.ebi.ac.uk:./3_ebi_submission/' % { + 'ebi_dir': e.full_ebi_dir}).split('\n') self.assertEqual(obs, exp) def test_parse_EBI_reply(self): ppd = self.generate_new_study_with_preprocessed_data() study_id = ppd.study e = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) e.generate_demultiplexed_fastq(mtime=1) e.generate_xml_files() curl_result = CURL_RESULT_FULL.format( @@ -677,7 +687,7 @@ def test_parse_EBI_reply(self): ppd = self.write_demux_files(PrepTemplate(1)) e = EBISubmission(ppd.id, 'ADD') - self.files_to_remove.append(e.ebi_dir) + self.files_to_remove.append(e.full_ebi_dir) # removing samples so test text is easier to read keys_to_del = ['1.SKD6.640190', '1.SKM6.640187', '1.SKD9.640182', '1.SKM8.640201', '1.SKM2.640199', '1.SKB3.640195'] diff --git a/qiita_ware/test/test_metadata_pipeline.py b/qiita_ware/test/test_metadata_pipeline.py index 2cab14696..7a8bbc850 100644 --- a/qiita_ware/test/test_metadata_pipeline.py +++ b/qiita_ware/test/test_metadata_pipeline.py @@ -68,7 +68,7 @@ def test_create_templates_from_qiime_mapping_file(self): self.assertEqual(set(obs_st.categories()), exp) exp = {"barcode", "primer", "center_name", "run_prefix", "platform", - "library_construction_protocol", + "library_construction_protocol", "instrument_model", "experiment_design_description"} self.assertEqual(set(obs_pt.categories()), exp) @@ -94,7 +94,7 @@ def test_create_templates_from_qiime_mapping_file_reverse_linker(self): self.assertEqual(set(obs_st.categories()), exp) exp = {"barcode", "primer", "center_name", "run_prefix", "platform", - "library_construction_protocol", + "library_construction_protocol", "instrument_model", "experiment_design_description", "reverselinkerprimer"} self.assertEqual(set(obs_pt.categories()), exp) @@ -105,38 +105,41 @@ def test_create_templates_from_qiime_mapping_file_error(self): QIIME_MAP = ( - "#SampleID\tBarcodeSequence\tLinkerPrimerSequence\t" + "#SampleID\tBarcodeSequence\tLinkerPrimerSequence\tinstrument_model\t" "physical_specimen_location\tphysical_specimen_remaining\tdna_extracted\t" "sample_type\thost_subject_id\tlatitude\tlongitude\ttaxon_id\t" "scientific_name\tcenter_name\trun_prefix\tplatform\t" "library_construction_protocol\texperiment_design_description\t" "collection_timestamp\tDescription\n" - "Sample1\tGTCCGCAAGTTA\tGTGCCAGCMGCCGCGGTAA\tUCSD\tTRUE\tTRUE\ttype1\t" - "NotIdentified\t4.1\t4.1\t9606\thomo sapiens\tANL\trp_1\tILLUMINA\t" - "protocol_1\tedd_1\t05/28/15 11:00\tDescription S1\n" - "Sample2\tCGTAGAGCTCTC\tGTGCCAGCMGCCGCGGTAA\tUCSD\tTRUE\tTRUE\ttype2\t" - "NotIdentified\t4.2\t4.2\t9606\thomo sapiens\tANL\trp_1\tILLUMINA\t" - "protocol_1\tedd_1\t05/28/15 11:00\tDescription S2\n" - "Sample3\tCCTCTGAGAGCT\tGTGCCAGCMGCCGCGGTAA\tUCSD\tTRUE\tTRUE\ttype3\t" - "NotIdentified\t4.3\t4.3\t9606\thomo sapiens\tANL\trp_2\tILLUMINA\t" - "protocol_1\tedd_1\t05/28/15 11:00\tDescription S3\n") + "Sample1\tGTCCGCAAGTTA\tGTGCCAGCMGCCGCGGTAA\tIllumina MiSeq\tUCSD\tTRUE\t" + "TRUE\ttype1\tNotIdentified\t4.1\t4.1\t9606\thomo sapiens\tANL\trp_1\t" + "ILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S1\n" + "Sample2\tCGTAGAGCTCTC\tGTGCCAGCMGCCGCGGTAA\tIllumina MiSeq\tUCSD\tTRUE\t" + "TRUE\ttype2\tNotIdentified\t4.2\t4.2\t9606\thomo sapiens\tANL\trp_1\t" + "ILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S2\n" + "Sample3\tCCTCTGAGAGCT\tGTGCCAGCMGCCGCGGTAA\tIllumina MiSeq\tUCSD\tTRUE\t" + "TRUE\ttype3\tNotIdentified\t4.3\t4.3\t9606\thomo sapiens\tANL\trp_2\t" + "ILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S3\n") QIIME_MAP_WITH_REVERSE_LINKER_PRIMER = ( "#SampleID\tBarcodeSequence\tLinkerPrimerSequence\tReverseLinkerPrimer\t" "physical_specimen_location\tphysical_specimen_remaining\tdna_extracted\t" "sample_type\thost_subject_id\tlatitude\tlongitude\ttaxon_id\t" - "scientific_name\tcenter_name\trun_prefix\tplatform\t" + "scientific_name\tcenter_name\trun_prefix\tplatform\tinstrument_model\t" "library_construction_protocol\texperiment_design_description\t" "collection_timestamp\tDescription\n" "Sample1\tGTCCGCAAGTTA\tGTGCCAGCMGCCGCGGTAA\tGTGCCAGCMGCCGCGGTAA\tUCSD\t" "TRUE\tTRUE\ttype1\tNotIdentified\t4.1\t4.1\t9606\thomo sapiens\tANL\t" - "rp_1\tILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S1\n" + "rp_1\tILLUMINA\tIllumina MiSeq\tprotocol_1\tedd_1\t05/28/15 11:00\t" + "Description S1\n" "Sample2\tCGTAGAGCTCTC\tGTGCCAGCMGCCGCGGTAA\tGTGCCAGCMGCCGCGGTAA\tUCSD\t" "TRUE\tTRUE\ttype2\tNotIdentified\t4.2\t4.2\t9606\thomo sapiens\tANL\t" - "rp_1\tILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S2\n" + "rp_1\tILLUMINA\tIllumina MiSeq\tprotocol_1\tedd_1\t05/28/15 11:00\t" + "Description S2\n" "Sample3\tCCTCTGAGAGCT\tGTGCCAGCMGCCGCGGTAA\tGTGCCAGCMGCCGCGGTAA\tUCSD\t" "TRUE\tTRUE\ttype3\tNotIdentified\t4.3\t4.3\t9606\thomo sapiens\tANL\t" - "rp_2\tILLUMINA\tprotocol_1\tedd_1\t05/28/15 11:00\tDescription S3\n") + "rp_2\tILLUMINA\tIllumina MiSeq\tprotocol_1\tedd_1\t05/28/15 11:00\t" + "Description S3\n") QIIME_MAP_ERROR = ( "#SampleID\tBarcodeSequence\tphysical_specimen_location\t" diff --git a/scripts/qiita b/scripts/qiita index 80251518f..4f4848906 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -354,10 +354,8 @@ def remove_study(portal, studies): @click.option('--send/--no-send', default=False, help="Controls whether or " "not sequence files and metadata will actually be sent to EBI " "(default is to generate all the files, but not to send)") -@click.option('--fastq-dir', type=click.Path(exists=True, file_okay=False, - resolve_path=True)) def submit(preprocessed_data_id, action, send, fastq_dir): - _submit_EBI(preprocessed_data_id, action, send, fastq_dir) + _submit_EBI(preprocessed_data_id, action, send) # ############################################################################# diff --git a/scripts/qiita-test-install b/scripts/qiita-test-install index 694ee8d57..3a01e1ff3 100755 --- a/scripts/qiita-test-install +++ b/scripts/qiita-test-install @@ -356,14 +356,12 @@ def main(): send_email = True except: send_email = False - ebi_credentials = (qiita_config.ebi_access_key != '' and - qiita_config.ebi_center_name != '' and + ebi_credentials = (qiita_config.ebi_center_name != '' and qiita_config.ebi_dropbox_url != '' and qiita_config.ebi_organization_prefix != '' and qiita_config.ebi_seq_xfer_pass != '' and qiita_config.ebi_seq_xfer_url != '' and - qiita_config.ebi_seq_xfer_user != '' and - qiita_config.ebi_skip_curl_cert != '') + qiita_config.ebi_seq_xfer_user != '') vamps_credentials = (qiita_config.vamps_pass != '' and qiita_config.vamps_url != '' and qiita_config.vamps_user != '') diff --git a/setup_acsp.sh b/setup_acsp.sh deleted file mode 100644 index d1e88024f..000000000 --- a/setup_acsp.sh +++ /dev/null @@ -1,4 +0,0 @@ -openssl aes-256-cbc -K $encrypted_e698cf0e691c_key -iv $encrypted_e698cf0e691c_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d; \ -curl -s -O ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh -ascp-install-3.5.4.102989-linux-64-qiita.sh