Merge pull request #1488 from antgonza/next-joy-list

WIP: Next steps

.travis.yml

@@ -15,7 +15,10 @@ before_install:
   # Update conda itself
   - conda update --yes conda
   # Downloading and setting up ascp for EBI testing
-  - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then sh setup_acsp.sh; fi
+  - wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh
+  - chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh
+  - ./ascp-install-3.5.4.102989-linux-64-qiita.sh
+  - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then openssl aes-256-cbc -K $encrypted_e698cf0e691c_key -iv $encrypted_e698cf0e691c_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d; fi
 install:
   # install a few of the dependencies that pip would otherwise try to install
   # when intalling scikit-bio
@@ -36,8 +39,7 @@ script:
   - qiita-env make --no-load-ontologies
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi
   - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi
-  - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage
-    ; fi
+  - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage; fi
   - if [ ${TEST_ADD_STUDIES} == "False" ]; then flake8 qiita_* setup.py scripts/qiita
     scripts/qiita-env scripts/qiita-test-install ; fi
   - ls -R /home/travis/miniconda3/envs/env_name/lib/python2.7/site-packages/qiita_pet/support_files/doc/

qiita_core/configuration_manager.py

@@ -78,16 +78,12 @@ class ConfigurationManager(object):
     smtp_email : str
         The email address that mail will be sent from when sending mail from
         the SMTP server
-    ebi_access_key : str
-        The access key issued by EBI for REST submissions
     ebi_seq_xfer_user : str
         The user to use when submitting to EBI
     ebi_seq_xfer_pass : str
         The password for the ebi_seq_xfer_user
     ebi_seq_xfer_url : str
         The URL of EBI's sequence portal site
-    ebi_skip_curl_cert : bool
-        Whether or not to skip the certificate check when curling the metadata
     ebi_center_name : str
         The name of the sequencing center to use when doing EBI submissions
     ebi_organization_prefix : str
@@ -213,14 +209,11 @@ def _get_smtp(self, config):
 
     def _get_ebi(self, config):
         sec_get = partial(config.get, 'ebi')
-        sec_getbool = partial(config.getboolean, 'ebi')
 
-        self.ebi_access_key = sec_get('EBI_ACCESS_KEY')
         self.ebi_seq_xfer_user = sec_get('EBI_SEQ_XFER_USER')
         self.ebi_seq_xfer_pass = sec_get('EBI_SEQ_XFER_PASS')
         self.ebi_seq_xfer_url = sec_get('EBI_SEQ_XFER_URL')
         self.ebi_dropbox_url = sec_get('EBI_DROPBOX_URL')
-        self.ebi_skip_curl_cert = sec_getbool('EBI_SKIP_CURL_CERT')
         self.ebi_center_name = sec_get('EBI_CENTER_NAME')
         self.ebi_organization_prefix = sec_get('EBI_ORGANIZATION_PREFIX')
 

qiita_core/support_files/config_test.cfg

@@ -99,9 +99,6 @@ ADMIN_PASSWORD =
 
 # ----------------------------- EBI settings -----------------------------
 [ebi]
-# The access key issued by EBI for REST submissions
-EBI_ACCESS_KEY =
-
 # The user to use when submitting to EBI
 EBI_SEQ_XFER_USER = Webin-41528
 
@@ -117,12 +114,8 @@ EBI_SEQ_XFER_URL = webin.ebi.ac.uk
 # testing URL
 EBI_DROPBOX_URL = https://www-test.ebi.ac.uk/ena/submit/drop-box/submit/
 
-# Skip acertificate check when calling curl (necessary for using the EBI test
-# dropbox URL)
-EBI_SKIP_CURL_CERT = True
-
 # The name of the sequencing center to use when doing EBI submissions
-EBI_CENTER_NAME = CCME-COLORADO
+EBI_CENTER_NAME = qiita-test
 
 # This string (with an underscore) will be prefixed to your EBI submission and
 # study aliases

qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst

@@ -38,6 +38,8 @@ These are the columns required for successfully submit your data to EBI:
 +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+
 | ``taxon_id``                     | integer              | NCBI's taxon id for the sample                                                                           |
 +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+
+| ``description``                  | free text            | Description of the sample.                                                                               |
++----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+
 | ``scientific_name``              | free text            | NCBI's scientific name for the provided taxon ID                                                         |
 +----------------------------------+----------------------+----------------------------------------------------------------------------------------------------------+
 
@@ -52,8 +54,6 @@ the centralized `Qiita server <http://qiita.microbio.me>`__:
 +===================================+======================================================================+=============================================================================================================================================================================================================================================+
 | ``sample_type``                   | free text                                                            | Description of the type of sample.                                                                                                                                                                                                          |
 +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| ``description``                   | free text                                                            | Description of the sample.                                                                                                                                                                                                                  |
-+-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``physical_specimen_remaining``   | ``y`` or ``n``                                                       | Is there still physical sample (e.g., soil, not DNA) available?                                                                                                                                                                             |
 +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``dna_extracted``                 | ``y`` or ``n``                                                       | Has DNA already been extracted for this sample?                                                                                                                                                                                             |

qiita_ware/commands.py

@@ -10,20 +10,20 @@
 from shutil import rmtree
 from tarfile import open as taropen
 from tempfile import mkdtemp
-
+from os import environ
+from traceback import format_exc
 from moi.job import system_call
+
 from qiita_db.study import Study
 from qiita_db.data import PreprocessedData
 from qiita_db.metadata_template import PrepTemplate, SampleTemplate
 from qiita_db.logger import LogEntry
 from qiita_core.qiita_settings import qiita_config
 from qiita_ware.ebi import EBISubmission
 from qiita_ware.exceptions import ComputeError, EBISubmissionError
-from traceback import format_exc
-from os import environ
 
 
-def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
+def submit_EBI(preprocessed_data_id, action, send):
     """Submit a preprocessed data to EBI
 
     Parameters
@@ -34,27 +34,19 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
         The action to perform with this data
     send : bool
         True to actually send the files
-    fastq_dir_fp : str, optional
-        The fastq filepath
-
-    Notes
-    -----
-    If fastq_dir_fp is passed, it must not contain any empty files, or
-    gzipped empty files
     """
     # step 1: init and validate
     ebi_submission = EBISubmission(preprocessed_data_id, action)
 
     # step 2: generate demux fastq files
-    ebi_submission.preprocessed_data.update_insdc_status('demuxing samples')
+    ebi_submission.study.ebi_submission_status = 'submitting'
     try:
         ebi_submission.generate_demultiplexed_fastq()
     except:
         error_msg = format_exc()
-        if isdir(ebi_submission.ebi_dir):
-            rmtree(ebi_submission.ebi_dir)
-        ebi_submission.preprocessed_data.update_insdc_status(
-            'failed: %s' % error_msg)
+        if isdir(ebi_submission.full_ebi_dir):
+            rmtree(ebi_submission.full_ebi_dir)
+        ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg
         LogEntry.create('Runtime', error_msg,
                         info={'ebi_submission': preprocessed_data_id})
         raise
@@ -66,49 +58,69 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
         # step 4: sending sequences
         old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
         environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
-        seqs_cmds = ebi_submission.generate_send_sequences_cmd()
+
         LogEntry.create('Runtime',
                         ("Submitting sequences for pre_processed_id: "
                          "%d" % preprocessed_data_id))
         try:
-            # place holder for moi call see #1477
-            pass
-        except:
-            LogEntry.create('Fatal', seqs_cmds,
-                            info={'ebi_submission': preprocessed_data_id})
-        else:
-            LogEntry.create('Runtime',
-                            ('Submission of sequences of pre_processed_id: '
-                             '%d completed successfully' %
-                             preprocessed_data_id))
-        environ['ASPERA_SCP_PASS'] = old_ascp_pass
+            for cmd in ebi_submission.generate_send_sequences_cmd():
+                try:
+                    stdout, stderr, _ = system_call(cmd)
+                except Exception as e:
+                    stdout = ''
+                    stderr = str(e)
+                    le = LogEntry.create(
+                        'Fatal', "Command: %s\nError: %s\n" % (cmd, str(e)),
+                        info={'ebi_submission': preprocessed_data_id})
+                    ebi_submission.study.ebi_submission_status = (
+                        "failed: ASCP submission, log id: %d" % le.id)
+                    raise ComputeError("EBI Submission failed! Log id: "
+                                       "%d" % le.id)
+                finally:
+                    open(ebi_submission.ascp_reply, 'a').write(
+                        'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
+        finally:
+            environ['ASPERA_SCP_PASS'] = old_ascp_pass
+        LogEntry.create('Runtime',
+                        ('Submission of sequences of pre_processed_id: '
+                         '%d completed successfully' %
+                         preprocessed_data_id))
 
         # step 5: sending xml and parsing answer
         xmls_cmds = ebi_submission.generate_curl_command()
         LogEntry.create('Runtime',
                         ("Submitting XMLs for pre_processed_id: "
                          "%d" % preprocessed_data_id))
         try:
-            # place holder for moi call see #1477
-            xmls_cmds_moi = xmls_cmds
-        except:
-            # handle exception
-            LogEntry.create('Fatal', seqs_cmds,
-                            info={'ebi_submission': preprocessed_data_id})
+            xml_content, stderr, _ = system_call(xmls_cmds)
+        except Exception as e:
+            xml_content = ''
+            stderr = str(e)
+            le = LogEntry.create('Fatal',
+                                 "Command: %s\nError: %s\n" % (
+                                    cmd, str(e)),
+                                 info={'ebi_submission': preprocessed_data_id})
+            ebi_submission.study.ebi_submission_status = (
+                "failed: XML submission, log id: %d" % le.id)
+            raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
         else:
             LogEntry.create('Runtime',
                             ('Submission of sequences of pre_processed_id: '
                              '%d completed successfully' %
                              preprocessed_data_id))
+        finally:
+            open(ebi_submission.curl_reply, 'w').write(
+                'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))
 
         try:
             st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
-                ebi_submission.parse_EBI_reply(xmls_cmds_moi)
+                ebi_submission.parse_EBI_reply(xml_content)
         except EBISubmissionError as e:
             le = LogEntry.create(
-                'Fatal', "Command: %s\nError: %s\n" % (xmls_cmds_moi, str(e)),
+                'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)),
                 info={'ebi_submission': preprocessed_data_id})
-            ebi_submission.preprocessed_data.update_insdc_status('failed')
+            ebi_submission.study.ebi_submission_status = (
+                "failed: XML parsing, log id: %d" % le.id)
             raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
 
         ebi_submission.study.ebi_submission_status = 'submitted'

qiita_ware/dispatchable.py

@@ -52,11 +52,7 @@ def preprocessor(study_id, prep_template_id, param_id, param_constructor):
 
 def submit_to_ebi(preprocessed_data_id, submission_type):
     """Submit a study to EBI"""
-    study_acc, submission_acc = submit_EBI(preprocessed_data_id,
-                                           submission_type,
-                                           True)
-
-    return study_acc, submission_acc
+    submit_EBI(preprocessed_data_id, submission_type, True)
 
 
 def submit_to_VAMPS(preprocessed_data_id):