Skip to content

Stop writing empty files and improve EBI tests #1172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
May 20, 2015
Merged

Stop writing empty files and improve EBI tests #1172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d6ece1a
Move ebi test files to subdirectory
May 8, 2015
1de34a8
Remove unused variable
May 8, 2015
1cc7f7e
Make fastq files actually look like fastq files
May 8, 2015
8702117
This directory does not need an __init__.py
May 8, 2015
1c3d2ad
Remove unused file
May 8, 2015
63fdb95
gzip test fastq files
May 8, 2015
a9229b9
Work with new files, fix paths
May 8, 2015
dd99ee5
Do not write empty per-sample fastq files
May 9, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion qiita_ware/commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# -----------------------------------------------------------------------------

from os.path import join, isdir
from os import makedirs
from os import makedirs, remove
from functools import partial
from tempfile import mkdtemp
from gzip import open as gzopen
Expand Down Expand Up @@ -43,6 +43,11 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
True to actually send the files
fastq_dir_fp : str, optional
The fastq filepath

Notes
-----
If fastq_dir_fp is passed, it must not contain any empty files, or
gzipped empty files
"""
preprocessed_data = PreprocessedData(preprocessed_data_id)
preprocessed_data_id_str = str(preprocessed_data_id)
Expand Down Expand Up @@ -99,9 +104,14 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
list(sample_template)):
demux_samples.add(samp)
sample_fp = join(fastq_dir_fp, "%s.fastq.gz" % samp)
wrote_sequences = False
with gzopen(sample_fp, 'w') as fh:
for record in iterator:
fh.write(record)
wrote_sequences = True

if not wrote_sequences:
remove(sample_fp)

output_dir = fastq_dir_fp + '_submission'

Expand Down
12 changes: 0 additions & 12 deletions qiita_ware/test/test_data/__init__.py
View file
Open in desktop

This file was deleted.

24 changes: 0 additions & 24 deletions qiita_ware/test/test_data/demux.fastq
View file
Open in desktop

This file was deleted.

1 change: 0 additions & 1 deletion qiita_ware/test/test_data/sample1.fastq
View file
Open in desktop

This file was deleted.

1 change: 0 additions & 1 deletion qiita_ware/test/test_data/sample2.fastq
View file
Open in desktop

This file was deleted.

1 change: 0 additions & 1 deletion qiita_ware/test/test_data/sample3.fastq
View file
Open in desktop

This file was deleted.

Binary file added qiita_ware/test/test_data/test_ebi/sample1.fastq.gz
View file
Open in desktop
Binary file not shown.
Binary file added qiita_ware/test/test_data/test_ebi/sample2.fastq.gz
View file
Open in desktop
Binary file not shown.
Binary file added qiita_ware/test/test_data/test_ebi/sample3.fastq.gz
View file
Open in desktop
Binary file not shown.
65 changes: 39 additions & 26 deletions qiita_ware/test/test_ebi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
from os import close, remove, path
from os.path import join
from tempfile import mkstemp, gettempdir
from shutil import rmtree
from unittest import TestCase, main
from xml.dom import minidom
from xml.etree import ElementTree as ET
from functools import partial

from qiita_ware.ebi import (SampleAlreadyExistsError, NoXMLError,
EBISubmission)
Expand All @@ -26,15 +26,16 @@

class TestEBISubmission(TestCase):
def setUp(self):
self.path = path.dirname(path.abspath(__file__)) + '/test_data'
self.temp_dir = gettempdir()
self.demux_output_dir = join(self.temp_dir, 'demux_output')
self.path = join(path.dirname(path.abspath(__file__)), 'test_data',
'test_ebi')

ebi_test_file = partial(join, self.path)

def tearDown(self):
try:
rmtree(self.demux_output_dir)
except:
pass
self.sample1_fp = ebi_test_file('sample1.fastq.gz')
self.sample2_fp = ebi_test_file('sample2.fastq.gz')
self.sample3_fp = ebi_test_file('sample3.fastq.gz')

self.temp_dir = gettempdir()

def test_init(self):
e = EBISubmission('2', 'Study Title', 'Study Abstract',
Expand Down Expand Up @@ -199,15 +200,18 @@ def test_add_sample_prep(self):
new_investigation_type='metagenome')
submission.add_sample('test1')
submission.add_sample('test2')

submission.add_sample_prep('test1', 'ILLUMINA', 'fastq',
self.path, 'experiment description',
self.sample1_fp, 'experiment description',
'library protocol')

prep_info = submission.samples['test1']['prep']
self.assertEqual(prep_info['platform'], 'ILLUMINA')
self.assertEqual(prep_info['file_path'], self.path)
self.assertEqual(prep_info['file_path'], self.sample1_fp)
with self.assertRaises(KeyError):
submission.add_sample_prep('test3', 'ILLUMINA', 'fastq',
self.path, 'experiment description',
self.sample3_fp,
'experiment description',
'library protocol')

def test_add_sample_prep_exception(self):
Expand All @@ -218,11 +222,13 @@ def test_add_sample_prep_exception(self):
submission.add_sample('test2')
with self.assertRaises(ValueError):
submission.add_sample_prep('test2', 'DOES-NOT-EXIST', 'fastq',
self.path, 'experiment description',
self.sample1_fp,
'experiment description',
'library protocol')
with self.assertRaises(KeyError):
submission.add_sample_prep('test3', 'DOES-NOT-EXIST', 'fastq',
self.path, 'experiment description',
self.sample3_fp,
'experiment description',
'library protocol')

def test_generate_library_descriptor(self):
Expand Down Expand Up @@ -253,14 +259,17 @@ def test_generate_experiment_xml(self):
new_investigation_type='metagenome')
submission.add_sample('test1')
submission.add_sample_prep('test1', 'ILLUMINA', 'fastq',
'fakepath',
self.sample1_fp,
'experiment description',
'library protocol')
xmlelement = submission.generate_experiment_xml()
xml = minidom.parseString(ET.tostring(xmlelement))
xmlstring = xml.toprettyxml(indent=' ', encoding='UTF-8')
obs_stripped = ''.join([l.strip() for l in xmlstring.splitlines()])
exp_stripped = ''.join([l.strip() for l in EXPERIMENTXML.splitlines()])
exp = EXPERIMENTXML % {
'path': self.sample1_fp,
'organization_prefix': qiita_config.ebi_organization_prefix}
exp_stripped = ''.join([l.strip() for l in exp.splitlines()])
self.assertEqual(obs_stripped, exp_stripped)

def test_generate_run_xml(self):
Expand All @@ -269,7 +278,7 @@ def test_generate_run_xml(self):
new_investigation_type='metagenome')
submission.add_sample('test1')
submission.add_sample_prep('test1', 'ILLUMINA', 'fastq',
join(self.path, '__init__.py'),
self.sample1_fp,
'experiment description',
'library protocol')
xmlelement = submission.generate_run_xml()
Expand Down Expand Up @@ -344,13 +353,16 @@ def test_write_experiment_xml(self):
new_investigation_type='metagenome')
submission.add_sample('test1')
submission.add_sample_prep('test1', 'ILLUMINA', 'fastq',
'fakepath', 'experiment description',
self.sample1_fp, 'experiment description',
'library protocol')
fh, output = mkstemp()
close(fh)
submission.write_experiment_xml(output)
obs_stripped = ''.join([l.strip() for l in open(output)])
exp_stripped = ''.join([l.strip() for l in EXPERIMENTXML.splitlines()])
exp = EXPERIMENTXML % {
'path': self.sample1_fp,
'organization_prefix': qiita_config.ebi_organization_prefix}
exp_stripped = ''.join([l.strip() for l in exp.splitlines()])
self.assertEqual(obs_stripped, exp_stripped)
remove(output)

Expand All @@ -369,7 +381,7 @@ def test_add_samples_from_templates(self):
'ILLUMINA')
self.assertEqual(
submission.samples['sample2']['prep']['file_path'],
self.path + '/sample2.fastq.gz')
self.sample2_fp)
with self.assertRaises(KeyError):
submission.samples['nothere']

Expand All @@ -394,7 +406,7 @@ def test_from_templates_and_per_sample_fastqs(self):
'ILLUMINA')
self.assertEqual(
submission.samples['sample2']['prep']['file_path'],
self.path + '/sample2.fastq.gz')
self.sample2_fp)
with self.assertRaises(KeyError):
submission.samples['nothere']

Expand Down Expand Up @@ -559,7 +571,7 @@ def test_generate_curl_command(self):
</EXPERIMENT_ATTRIBUTE>
<EXPERIMENT_ATTRIBUTE>
<TAG>file_path</TAG>
<VALUE>fakepath</VALUE>
<VALUE>%(path)s</VALUE>
</EXPERIMENT_ATTRIBUTE>
<EXPERIMENT_ATTRIBUTE>
<TAG>file_type</TAG>
Expand All @@ -576,18 +588,19 @@ def test_generate_curl_command(self):
</EXPERIMENT_ATTRIBUTES>
</EXPERIMENT>
</EXPERIMENT_SET>
""" % {'organization_prefix': qiita_config.ebi_organization_prefix}
"""

RUNXML = """
<?xml version="1.0" encoding="UTF-8"?>
<RUN_SET xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:no\
NamespaceSchemaLocation="ftp://ftp.sra.ebi.ac.uk/meta/xsd/sra_1_3/SRA.run.xsd">
<RUN alias="%(study_alias)s___init__.py_run" center_name="CCME-COLORADO">
<RUN alias="%(study_alias)s_sample1.fastq.gz_run" \
center_name="CCME-COLORADO">
<EXPERIMENT_REF refname="%(organization_prefix)s_ppdid_001:test1"/>
<DATA_BLOCK>
<FILES>
<FILE checksum="612cbff13a4f0e236e5e62ac2e00329a" checksum_method=\
"MD5" filename="%(ebi_dir)s/__init__.py" filetype="fastq" \
<FILE checksum="506d31c82999a2cbcda138a369955e7d" checksum_method=\
"MD5" filename="%(ebi_dir)s/sample1.fastq.gz" filetype="fastq" \
quality_scoring_system="phred"/>
</FILES>
</DATA_BLOCK>
Expand Down