-
Notifications
You must be signed in to change notification settings - Fork 80
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
WIP: Next steps #1488
WIP: Next steps #1488
Changes from 13 commits
2215406
c0c3caa
42b73ed
72cac32
52f6496
29c945e
c45be2e
b2be7e0
1928106
1c7ef8e
fe1070d
6417aa6
abc5fe8
13f80b1
0586d8b
30723c2
f57a364
a7f7dfe
9377a3a
d2eae77
a368f41
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -10,6 +10,10 @@ | |
from shutil import rmtree | ||
from tarfile import open as taropen | ||
from tempfile import mkdtemp | ||
from os import environ | ||
from traceback import format_exc | ||
from subprocess import call | ||
from shlex import split as shsplit | ||
|
||
from moi.job import system_call | ||
from qiita_db.study import Study | ||
|
@@ -19,11 +23,9 @@ | |
from qiita_core.qiita_settings import qiita_config | ||
from qiita_ware.ebi import EBISubmission | ||
from qiita_ware.exceptions import ComputeError, EBISubmissionError | ||
from traceback import format_exc | ||
from os import environ | ||
|
||
|
||
def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): | ||
def submit_EBI(preprocessed_data_id, action, send): | ||
"""Submit a preprocessed data to EBI | ||
|
||
Parameters | ||
|
@@ -34,27 +36,19 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): | |
The action to perform with this data | ||
send : bool | ||
True to actually send the files | ||
fastq_dir_fp : str, optional | ||
The fastq filepath | ||
|
||
Notes | ||
----- | ||
If fastq_dir_fp is passed, it must not contain any empty files, or | ||
gzipped empty files | ||
""" | ||
# step 1: init and validate | ||
ebi_submission = EBISubmission(preprocessed_data_id, action) | ||
|
||
# step 2: generate demux fastq files | ||
ebi_submission.preprocessed_data.update_insdc_status('demuxing samples') | ||
ebi_submission.study.ebi_submission_status = 'submitting' | ||
try: | ||
ebi_submission.generate_demultiplexed_fastq() | ||
except: | ||
error_msg = format_exc() | ||
if isdir(ebi_submission.ebi_dir): | ||
rmtree(ebi_submission.ebi_dir) | ||
ebi_submission.preprocessed_data.update_insdc_status( | ||
'failed: %s' % error_msg) | ||
if isdir(ebi_submission.full_ebi_dir): | ||
rmtree(ebi_submission.full_ebi_dir) | ||
ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg | ||
LogEntry.create('Runtime', error_msg, | ||
info={'ebi_submission': preprocessed_data_id}) | ||
raise | ||
|
@@ -66,34 +60,42 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): | |
# step 4: sending sequences | ||
old_ascp_pass = environ.get('ASPERA_SCP_PASS', '') | ||
environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass | ||
seqs_cmds = ebi_submission.generate_send_sequences_cmd() | ||
|
||
LogEntry.create('Runtime', | ||
("Submitting sequences for pre_processed_id: " | ||
"%d" % preprocessed_data_id)) | ||
try: | ||
# place holder for moi call see #1477 | ||
pass | ||
except: | ||
LogEntry.create('Fatal', seqs_cmds, | ||
info={'ebi_submission': preprocessed_data_id}) | ||
else: | ||
LogEntry.create('Runtime', | ||
('Submission of sequences of pre_processed_id: ' | ||
'%d completed successfully' % | ||
preprocessed_data_id)) | ||
for cmd in ebi_submission.generate_send_sequences_cmd(): | ||
cmd_pieces = shsplit(cmd) | ||
try: | ||
call(cmd_pieces, stdout=open(ebi_submission.ascp_reply, 'a')) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it worth using moi's |
||
except: | ||
with open(ebi_submission.ascp_reply, 'r') as f: | ||
content = f.read() | ||
ebi_submission.study.ebi_submission_status = ( | ||
"failed: ASCP - %s" % content) | ||
LogEntry.create('Fatal', content, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Are we in risk of this content being too large? i.e. for a submission with thousands of samples? If that's the case, do we really want that much text in the log table? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The only time I have seen this fail is when the login is incorrect and the reply is something like: "Wong login" or something like that. So, no? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Cool, just wanted to avoid cluttering the logging page via the web On (Oct-09-15|10:18), Antonio Gonzalez wrote:
|
||
info={ | ||
'ebi_submission': preprocessed_data_id, | ||
'fail': cmd}) | ||
LogEntry.create('Runtime', | ||
('Submission of sequences of pre_processed_id: ' | ||
'%d completed successfully' % | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does this activate the fireworks at the top of BRF2? 😃 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nop, cause this time it actually works well (or at least we hope!) vs. giving you the impression via fireworks that it does. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So the lack of fireworks is the celebration of success, as this is real On (Oct-09-15|10:19), Antonio Gonzalez wrote:
|
||
preprocessed_data_id)) | ||
environ['ASPERA_SCP_PASS'] = old_ascp_pass | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the above code should be in a try/except so this line can be in a finally. If for whatever reason there is an exception on the above block of code, the environment variable 'ASPERA_SCP_PASS' will not reset. |
||
|
||
# step 5: sending xml and parsing answer | ||
xmls_cmds = ebi_submission.generate_curl_command() | ||
xmls_cmds = shsplit(ebi_submission.generate_curl_command()) | ||
LogEntry.create('Runtime', | ||
("Submitting XMLs for pre_processed_id: " | ||
"%d" % preprocessed_data_id)) | ||
try: | ||
# place holder for moi call see #1477 | ||
xmls_cmds_moi = xmls_cmds | ||
call(xmls_cmds, stdout=open(ebi_submission.curl_reply, 'w')) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it may be useful to store the stderr too, in case that something unexpected happens (not only in EBI end) |
||
with open(ebi_submission.curl_reply, 'r') as f: | ||
xml_content = f.read() | ||
except: | ||
# handle exception | ||
LogEntry.create('Fatal', seqs_cmds, | ||
with open(ebi_submission.curl_reply, 'r') as f: | ||
xml_content = f.read() | ||
LogEntry.create('Fatal', xml_content, | ||
info={'ebi_submission': preprocessed_data_id}) | ||
else: | ||
LogEntry.create('Runtime', | ||
|
@@ -103,12 +105,13 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): | |
|
||
try: | ||
st_acc, sa_acc, bio_acc, ex_acc, run_acc = \ | ||
ebi_submission.parse_EBI_reply(xmls_cmds_moi) | ||
ebi_submission.parse_EBI_reply(xml_content) | ||
except EBISubmissionError as e: | ||
le = LogEntry.create( | ||
'Fatal', "Command: %s\nError: %s\n" % (xmls_cmds_moi, str(e)), | ||
'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)), | ||
info={'ebi_submission': preprocessed_data_id}) | ||
ebi_submission.preprocessed_data.update_insdc_status('failed') | ||
ebi_submission.study.ebi_submission_status = ( | ||
"failed: XML submission, log id: %d" % le.id) | ||
raise ComputeError("EBI Submission failed! Log id: %d" % le.id) | ||
|
||
ebi_submission.study.ebi_submission_status = 'submitted' | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The rendering is a little bit off, would you mind having a look at the formatting?