-
Notifications
You must be signed in to change notification settings - Fork 80
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
WIP: Next steps #1488
WIP: Next steps #1488
Changes from 15 commits
2215406
c0c3caa
42b73ed
72cac32
52f6496
29c945e
c45be2e
b2be7e0
1928106
1c7ef8e
fe1070d
6417aa6
abc5fe8
13f80b1
0586d8b
30723c2
f57a364
a7f7dfe
9377a3a
d2eae77
a368f41
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -10,20 +10,20 @@ | |
from shutil import rmtree | ||
from tarfile import open as taropen | ||
from tempfile import mkdtemp | ||
|
||
from os import environ | ||
from traceback import format_exc | ||
from moi.job import system_call | ||
|
||
from qiita_db.study import Study | ||
from qiita_db.data import PreprocessedData | ||
from qiita_db.metadata_template import PrepTemplate, SampleTemplate | ||
from qiita_db.logger import LogEntry | ||
from qiita_core.qiita_settings import qiita_config | ||
from qiita_ware.ebi import EBISubmission | ||
from qiita_ware.exceptions import ComputeError, EBISubmissionError | ||
from traceback import format_exc | ||
from os import environ | ||
|
||
|
||
def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): | ||
def submit_EBI(preprocessed_data_id, action, send): | ||
"""Submit a preprocessed data to EBI | ||
|
||
Parameters | ||
|
@@ -34,27 +34,19 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): | |
The action to perform with this data | ||
send : bool | ||
True to actually send the files | ||
fastq_dir_fp : str, optional | ||
The fastq filepath | ||
|
||
Notes | ||
----- | ||
If fastq_dir_fp is passed, it must not contain any empty files, or | ||
gzipped empty files | ||
""" | ||
# step 1: init and validate | ||
ebi_submission = EBISubmission(preprocessed_data_id, action) | ||
|
||
# step 2: generate demux fastq files | ||
ebi_submission.preprocessed_data.update_insdc_status('demuxing samples') | ||
ebi_submission.study.ebi_submission_status = 'submitting' | ||
try: | ||
ebi_submission.generate_demultiplexed_fastq() | ||
except: | ||
error_msg = format_exc() | ||
if isdir(ebi_submission.ebi_dir): | ||
rmtree(ebi_submission.ebi_dir) | ||
ebi_submission.preprocessed_data.update_insdc_status( | ||
'failed: %s' % error_msg) | ||
if isdir(ebi_submission.full_ebi_dir): | ||
rmtree(ebi_submission.full_ebi_dir) | ||
ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg | ||
LogEntry.create('Runtime', error_msg, | ||
info={'ebi_submission': preprocessed_data_id}) | ||
raise | ||
|
@@ -66,49 +58,65 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): | |
# step 4: sending sequences | ||
old_ascp_pass = environ.get('ASPERA_SCP_PASS', '') | ||
environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass | ||
seqs_cmds = ebi_submission.generate_send_sequences_cmd() | ||
|
||
LogEntry.create('Runtime', | ||
("Submitting sequences for pre_processed_id: " | ||
"%d" % preprocessed_data_id)) | ||
try: | ||
# place holder for moi call see #1477 | ||
pass | ||
except: | ||
LogEntry.create('Fatal', seqs_cmds, | ||
info={'ebi_submission': preprocessed_data_id}) | ||
else: | ||
LogEntry.create('Runtime', | ||
('Submission of sequences of pre_processed_id: ' | ||
'%d completed successfully' % | ||
preprocessed_data_id)) | ||
environ['ASPERA_SCP_PASS'] = old_ascp_pass | ||
for cmd in ebi_submission.generate_send_sequences_cmd(): | ||
try: | ||
stdout, stderr, _ = system_call(cmd) | ||
except Exception as e: | ||
stdout = '' | ||
stderr = str(e) | ||
ebi_submission.study.ebi_submission_status = ( | ||
"failed: ASCP - %s" % stderr) | ||
LogEntry.create('Fatal', stderr, | ||
info={ | ||
'ebi_submission': preprocessed_data_id, | ||
'fail': cmd}) | ||
raise | ||
finally: | ||
open(ebi_submission.ascp_reply, 'a').write( | ||
'stdout:\n%s\n\nstderr: %s' % (stdout, stderr)) | ||
finally: | ||
environ['ASPERA_SCP_PASS'] = old_ascp_pass | ||
LogEntry.create('Runtime', | ||
('Submission of sequences of pre_processed_id: ' | ||
'%d completed successfully' % | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does this activate the fireworks at the top of BRF2? 😃 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nop, cause this time it actually works well (or at least we hope!) vs. giving you the impression via fireworks that it does. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So the lack of fireworks is the celebration of success, as this is real On (Oct-09-15|10:19), Antonio Gonzalez wrote:
|
||
preprocessed_data_id)) | ||
|
||
# step 5: sending xml and parsing answer | ||
xmls_cmds = ebi_submission.generate_curl_command() | ||
LogEntry.create('Runtime', | ||
("Submitting XMLs for pre_processed_id: " | ||
"%d" % preprocessed_data_id)) | ||
try: | ||
# place holder for moi call see #1477 | ||
xmls_cmds_moi = xmls_cmds | ||
except: | ||
# handle exception | ||
LogEntry.create('Fatal', seqs_cmds, | ||
info={'ebi_submission': preprocessed_data_id}) | ||
xml_content, stderr, _ = system_call(xmls_cmds) | ||
|
||
except Exception as e: | ||
error = str(e) | ||
LogEntry.create('Fatal', error, | ||
info={'ebi_submission': preprocessed_data_id, | ||
'fail': cmd}) | ||
else: | ||
LogEntry.create('Runtime', | ||
('Submission of sequences of pre_processed_id: ' | ||
'%d completed successfully' % | ||
preprocessed_data_id)) | ||
finally: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If |
||
open(ebi_submission.curl_reply, 'w').write( | ||
'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr)) | ||
|
||
try: | ||
st_acc, sa_acc, bio_acc, ex_acc, run_acc = \ | ||
ebi_submission.parse_EBI_reply(xmls_cmds_moi) | ||
ebi_submission.parse_EBI_reply(xml_content) | ||
except EBISubmissionError as e: | ||
le = LogEntry.create( | ||
'Fatal', "Command: %s\nError: %s\n" % (xmls_cmds_moi, str(e)), | ||
'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)), | ||
info={'ebi_submission': preprocessed_data_id}) | ||
ebi_submission.preprocessed_data.update_insdc_status('failed') | ||
ebi_submission.study.ebi_submission_status = ( | ||
"failed: XML submission, log id: %d" % le.id) | ||
raise ComputeError("EBI Submission failed! Log id: %d" % le.id) | ||
|
||
ebi_submission.study.ebi_submission_status = 'submitted' | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The rendering is a little bit off, would you mind having a look at the formatting?