From d6ece1ad24121f42798a6e84637425b592f0f369 Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 15:41:10 -0600 Subject: [PATCH 1/8] Move ebi test files to subdirectory --- qiita_ware/test/test_data/{ => test_ebi}/sample1.fastq | 0 qiita_ware/test/test_data/{ => test_ebi}/sample2.fastq | 0 qiita_ware/test/test_data/{ => test_ebi}/sample3.fastq | 0 qiita_ware/test/test_ebi.py | 3 ++- 4 files changed, 2 insertions(+), 1 deletion(-) rename qiita_ware/test/test_data/{ => test_ebi}/sample1.fastq (100%) rename qiita_ware/test/test_data/{ => test_ebi}/sample2.fastq (100%) rename qiita_ware/test/test_data/{ => test_ebi}/sample3.fastq (100%) diff --git a/qiita_ware/test/test_data/sample1.fastq b/qiita_ware/test/test_data/test_ebi/sample1.fastq similarity index 100% rename from qiita_ware/test/test_data/sample1.fastq rename to qiita_ware/test/test_data/test_ebi/sample1.fastq diff --git a/qiita_ware/test/test_data/sample2.fastq b/qiita_ware/test/test_data/test_ebi/sample2.fastq similarity index 100% rename from qiita_ware/test/test_data/sample2.fastq rename to qiita_ware/test/test_data/test_ebi/sample2.fastq diff --git a/qiita_ware/test/test_data/sample3.fastq b/qiita_ware/test/test_data/test_ebi/sample3.fastq similarity index 100% rename from qiita_ware/test/test_data/sample3.fastq rename to qiita_ware/test/test_data/test_ebi/sample3.fastq diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 3abe3dba9..6fe16b801 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -26,7 +26,8 @@ class TestEBISubmission(TestCase): def setUp(self): - self.path = path.dirname(path.abspath(__file__)) + '/test_data' + self.path = path.dirname(path.abspath(__file__)) + \ + '/test_data/test_ebi' self.temp_dir = gettempdir() self.demux_output_dir = join(self.temp_dir, 'demux_output') From 1de34a89ae8e3f5022f4f6468ff2fd0f80047145 Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 15:41:33 -0600 Subject: [PATCH 2/8] Remove unused variable --- qiita_ware/test/test_ebi.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 6fe16b801..f286d3d8b 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -29,13 +29,6 @@ def setUp(self): self.path = path.dirname(path.abspath(__file__)) + \ '/test_data/test_ebi' self.temp_dir = gettempdir() - self.demux_output_dir = join(self.temp_dir, 'demux_output') - - def tearDown(self): - try: - rmtree(self.demux_output_dir) - except: - pass def test_init(self): e = EBISubmission('2', 'Study Title', 'Study Abstract', From 1cc7f7e5beda0b476ec27bbf59e778fad3c11b87 Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 15:42:04 -0600 Subject: [PATCH 3/8] Make fastq files actually look like fastq files --- qiita_ware/test/test_data/test_ebi/sample1.fastq | 5 ++++- qiita_ware/test/test_data/test_ebi/sample2.fastq | 5 ++++- qiita_ware/test/test_data/test_ebi/sample3.fastq | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qiita_ware/test/test_data/test_ebi/sample1.fastq b/qiita_ware/test/test_data/test_ebi/sample1.fastq index 9868291fd..092a225c7 100644 --- a/qiita_ware/test/test_data/test_ebi/sample1.fastq +++ b/qiita_ware/test/test_data/test_ebi/sample1.fastq @@ -1 +1,4 @@ ->sample1 +@sample1 +ATGACG ++ +FFFFFF diff --git a/qiita_ware/test/test_data/test_ebi/sample2.fastq b/qiita_ware/test/test_data/test_ebi/sample2.fastq index 16638396d..1cc05da9d 100644 --- a/qiita_ware/test/test_data/test_ebi/sample2.fastq +++ b/qiita_ware/test/test_data/test_ebi/sample2.fastq @@ -1 +1,4 @@ ->sample2 +@sample2 +ATGACG ++ +FFFFFF diff --git a/qiita_ware/test/test_data/test_ebi/sample3.fastq b/qiita_ware/test/test_data/test_ebi/sample3.fastq index ea65e1bf9..aba58e6fa 100644 --- a/qiita_ware/test/test_data/test_ebi/sample3.fastq +++ b/qiita_ware/test/test_data/test_ebi/sample3.fastq @@ -1 +1,4 @@ ->sample3 +@sample3 +ATGACG ++ +FFFFFF From 87021172a9574aef3ed3e27344a6e7c65709d0ec Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 15:42:27 -0600 Subject: [PATCH 4/8] This directory does not need an __init__.py Nothing is imported from this directory --- qiita_ware/test/test_data/__init__.py | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 qiita_ware/test/test_data/__init__.py diff --git a/qiita_ware/test/test_data/__init__.py b/qiita_ware/test/test_data/__init__.py deleted file mode 100644 index 744315f60..000000000 --- a/qiita_ware/test/test_data/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import division - -# ----------------------------------------------------------------------------- -# Copyright (c) 2014--, The Qiita Development Team. -# -# Distributed under the terms of the BSD 3-clause License. -# -# The full license is in the file LICENSE, distributed with this software. -# ----------------------------------------------------------------------------- - -# Do not modify this file as TestEBISubmission.test_generate_run_xml depends -# on this file having a constant MD5 sum. From 1c3d2ad82def4fb9435cab3bf9c31d0922af6423 Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 15:44:07 -0600 Subject: [PATCH 5/8] Remove unused file --- qiita_ware/test/test_data/demux.fastq | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 qiita_ware/test/test_data/demux.fastq diff --git a/qiita_ware/test/test_data/demux.fastq b/qiita_ware/test/test_data/demux.fastq deleted file mode 100644 index ac9efe4ac..000000000 --- a/qiita_ware/test/test_data/demux.fastq +++ /dev/null @@ -1,24 +0,0 @@ -@sample1_1 -ATC -+ -ABC -@sample1_2 -AGT -+ -ABC -@sample2_1 -GCT -+ -ABC -@sample2_2 -CGT -+ -ABC -@sample3_1 -AAA -+ -ABC -@sample3_2 -AAA -+ -ABC From 63fdb95328835bdf97a6ab88f26dd910cf09359d Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 17:06:05 -0600 Subject: [PATCH 6/8] gzip test fastq files --- qiita_ware/test/test_data/test_ebi/sample1.fastq | 4 ---- .../test/test_data/test_ebi/sample1.fastq.gz | Bin 0 -> 55 bytes qiita_ware/test/test_data/test_ebi/sample2.fastq | 4 ---- .../test/test_data/test_ebi/sample2.fastq.gz | Bin 0 -> 55 bytes qiita_ware/test/test_data/test_ebi/sample3.fastq | 4 ---- .../test/test_data/test_ebi/sample3.fastq.gz | Bin 0 -> 55 bytes 6 files changed, 12 deletions(-) delete mode 100644 qiita_ware/test/test_data/test_ebi/sample1.fastq create mode 100644 qiita_ware/test/test_data/test_ebi/sample1.fastq.gz delete mode 100644 qiita_ware/test/test_data/test_ebi/sample2.fastq create mode 100644 qiita_ware/test/test_data/test_ebi/sample2.fastq.gz delete mode 100644 qiita_ware/test/test_data/test_ebi/sample3.fastq create mode 100644 qiita_ware/test/test_data/test_ebi/sample3.fastq.gz diff --git a/qiita_ware/test/test_data/test_ebi/sample1.fastq b/qiita_ware/test/test_data/test_ebi/sample1.fastq deleted file mode 100644 index 092a225c7..000000000 --- a/qiita_ware/test/test_data/test_ebi/sample1.fastq +++ /dev/null @@ -1,4 +0,0 @@ -@sample1 -ATGACG -+ -FFFFFF diff --git a/qiita_ware/test/test_data/test_ebi/sample1.fastq.gz b/qiita_ware/test/test_data/test_ebi/sample1.fastq.gz new file mode 100644 index 0000000000000000000000000000000000000000..2df9c489ce4de6329764e4770f81f2c14a63986c GIT binary patch literal 55 zcmb2|=HLj@@eO5QE>6rX$VoNSOG_*+DP$!qfCg5l>-xN!jyD&x)9t^%ySk LNPm@NU|;|M_GuD^ literal 0 HcmV?d00001 diff --git a/qiita_ware/test/test_data/test_ebi/sample2.fastq b/qiita_ware/test/test_data/test_ebi/sample2.fastq deleted file mode 100644 index 1cc05da9d..000000000 --- a/qiita_ware/test/test_data/test_ebi/sample2.fastq +++ /dev/null @@ -1,4 +0,0 @@ -@sample2 -ATGACG -+ -FFFFFF diff --git a/qiita_ware/test/test_data/test_ebi/sample2.fastq.gz b/qiita_ware/test/test_data/test_ebi/sample2.fastq.gz new file mode 100644 index 0000000000000000000000000000000000000000..cef3ddb66f2ec2b689adb7418063ad81ddbd20f9 GIT binary patch literal 55 zcmb2|=HO`3@eO5QE>6rX$VoNQOG_*+DP$!qfCo5l>-xN!jyD&x)9t^%yo> LoGUNMz`y_i12Pi? literal 0 HcmV?d00001 diff --git a/qiita_ware/test/test_data/test_ebi/sample3.fastq b/qiita_ware/test/test_data/test_ebi/sample3.fastq deleted file mode 100644 index aba58e6fa..000000000 --- a/qiita_ware/test/test_data/test_ebi/sample3.fastq +++ /dev/null @@ -1,4 +0,0 @@ -@sample3 -ATGACG -+ -FFFFFF diff --git a/qiita_ware/test/test_data/test_ebi/sample3.fastq.gz b/qiita_ware/test/test_data/test_ebi/sample3.fastq.gz new file mode 100644 index 0000000000000000000000000000000000000000..ee070043c45fab2de289fec9e4bdd2ea892b455c GIT binary patch literal 55 zcmb2|=HO`3@eO5QE>6rX$VoNUOG_*+DP$!qfCw5l>-xN!jyD&x)9t^%w%v L&wr3)U|;|M0~ixS literal 0 HcmV?d00001 From a9229b9853c47e8c4afba920cb9464dfd460864e Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 17:39:18 -0600 Subject: [PATCH 7/8] Work with new files, fix paths --- qiita_ware/test/test_ebi.py | 61 +++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index f286d3d8b..864384a50 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -18,6 +18,7 @@ from unittest import TestCase, main from xml.dom import minidom from xml.etree import ElementTree as ET +from functools import partial from qiita_ware.ebi import (SampleAlreadyExistsError, NoXMLError, EBISubmission) @@ -26,8 +27,15 @@ class TestEBISubmission(TestCase): def setUp(self): - self.path = path.dirname(path.abspath(__file__)) + \ - '/test_data/test_ebi' + self.path = join(path.dirname(path.abspath(__file__)), 'test_data', + 'test_ebi') + + ebi_test_file = partial(join, self.path) + + self.sample1_fp = ebi_test_file('sample1.fastq.gz') + self.sample2_fp = ebi_test_file('sample2.fastq.gz') + self.sample3_fp = ebi_test_file('sample3.fastq.gz') + self.temp_dir = gettempdir() def test_init(self): @@ -193,15 +201,18 @@ def test_add_sample_prep(self): new_investigation_type='metagenome') submission.add_sample('test1') submission.add_sample('test2') + submission.add_sample_prep('test1', 'ILLUMINA', 'fastq', - self.path, 'experiment description', + self.sample1_fp, 'experiment description', 'library protocol') + prep_info = submission.samples['test1']['prep'] self.assertEqual(prep_info['platform'], 'ILLUMINA') - self.assertEqual(prep_info['file_path'], self.path) + self.assertEqual(prep_info['file_path'], self.sample1_fp) with self.assertRaises(KeyError): submission.add_sample_prep('test3', 'ILLUMINA', 'fastq', - self.path, 'experiment description', + self.sample3_fp, + 'experiment description', 'library protocol') def test_add_sample_prep_exception(self): @@ -212,11 +223,13 @@ def test_add_sample_prep_exception(self): submission.add_sample('test2') with self.assertRaises(ValueError): submission.add_sample_prep('test2', 'DOES-NOT-EXIST', 'fastq', - self.path, 'experiment description', + self.sample1_fp, + 'experiment description', 'library protocol') with self.assertRaises(KeyError): submission.add_sample_prep('test3', 'DOES-NOT-EXIST', 'fastq', - self.path, 'experiment description', + self.sample3_fp, + 'experiment description', 'library protocol') def test_generate_library_descriptor(self): @@ -247,14 +260,17 @@ def test_generate_experiment_xml(self): new_investigation_type='metagenome') submission.add_sample('test1') submission.add_sample_prep('test1', 'ILLUMINA', 'fastq', - 'fakepath', + self.sample1_fp, 'experiment description', 'library protocol') xmlelement = submission.generate_experiment_xml() xml = minidom.parseString(ET.tostring(xmlelement)) xmlstring = xml.toprettyxml(indent=' ', encoding='UTF-8') obs_stripped = ''.join([l.strip() for l in xmlstring.splitlines()]) - exp_stripped = ''.join([l.strip() for l in EXPERIMENTXML.splitlines()]) + exp = EXPERIMENTXML % { + 'path': self.sample1_fp, + 'organization_prefix': qiita_config.ebi_organization_prefix} + exp_stripped = ''.join([l.strip() for l in exp.splitlines()]) self.assertEqual(obs_stripped, exp_stripped) def test_generate_run_xml(self): @@ -263,7 +279,7 @@ def test_generate_run_xml(self): new_investigation_type='metagenome') submission.add_sample('test1') submission.add_sample_prep('test1', 'ILLUMINA', 'fastq', - join(self.path, '__init__.py'), + self.sample1_fp, 'experiment description', 'library protocol') xmlelement = submission.generate_run_xml() @@ -338,13 +354,16 @@ def test_write_experiment_xml(self): new_investigation_type='metagenome') submission.add_sample('test1') submission.add_sample_prep('test1', 'ILLUMINA', 'fastq', - 'fakepath', 'experiment description', + self.sample1_fp, 'experiment description', 'library protocol') fh, output = mkstemp() close(fh) submission.write_experiment_xml(output) obs_stripped = ''.join([l.strip() for l in open(output)]) - exp_stripped = ''.join([l.strip() for l in EXPERIMENTXML.splitlines()]) + exp = EXPERIMENTXML % { + 'path': self.sample1_fp, + 'organization_prefix': qiita_config.ebi_organization_prefix} + exp_stripped = ''.join([l.strip() for l in exp.splitlines()]) self.assertEqual(obs_stripped, exp_stripped) remove(output) @@ -363,7 +382,7 @@ def test_add_samples_from_templates(self): 'ILLUMINA') self.assertEqual( submission.samples['sample2']['prep']['file_path'], - self.path + '/sample2.fastq.gz') + self.sample2_fp) with self.assertRaises(KeyError): submission.samples['nothere'] @@ -388,7 +407,7 @@ def test_from_templates_and_per_sample_fastqs(self): 'ILLUMINA') self.assertEqual( submission.samples['sample2']['prep']['file_path'], - self.path + '/sample2.fastq.gz') + self.sample2_fp) with self.assertRaises(KeyError): submission.samples['nothere'] @@ -553,7 +572,7 @@ def test_generate_curl_command(self): file_path - fakepath + %(path)s file_type @@ -563,6 +582,10 @@ def test_generate_curl_command(self): library_construction_protocol library protocol + + md5 + 506d31c82999a2cbcda138a369955e7d + platform ILLUMINA @@ -570,18 +593,18 @@ def test_generate_curl_command(self): -""" % {'organization_prefix': qiita_config.ebi_organization_prefix} +""" RUNXML = """ - + - From dd99ee5ca4d87bc1a21f11c585b0202d0b48ca98 Mon Sep 17 00:00:00 2001 From: Adam Robbins-Pianka Date: Fri, 8 May 2015 18:03:01 -0600 Subject: [PATCH 8/8] Do not write empty per-sample fastq files --- qiita_ware/commands.py | 12 +++++++++++- qiita_ware/test/test_ebi.py | 8 ++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/qiita_ware/commands.py b/qiita_ware/commands.py index dc8928022..5ff3cf88f 100644 --- a/qiita_ware/commands.py +++ b/qiita_ware/commands.py @@ -7,7 +7,7 @@ # ----------------------------------------------------------------------------- from os.path import join, isdir -from os import makedirs +from os import makedirs, remove from functools import partial from tempfile import mkdtemp from gzip import open as gzopen @@ -43,6 +43,11 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): True to actually send the files fastq_dir_fp : str, optional The fastq filepath + + Notes + ----- + If fastq_dir_fp is passed, it must not contain any empty files, or + gzipped empty files """ preprocessed_data = PreprocessedData(preprocessed_data_id) preprocessed_data_id_str = str(preprocessed_data_id) @@ -99,9 +104,14 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None): list(sample_template)): demux_samples.add(samp) sample_fp = join(fastq_dir_fp, "%s.fastq.gz" % samp) + wrote_sequences = False with gzopen(sample_fp, 'w') as fh: for record in iterator: fh.write(record) + wrote_sequences = True + + if not wrote_sequences: + remove(sample_fp) output_dir = fastq_dir_fp + '_submission' diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 864384a50..372de4bd0 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -14,7 +14,6 @@ from os import close, remove, path from os.path import join from tempfile import mkstemp, gettempdir -from shutil import rmtree from unittest import TestCase, main from xml.dom import minidom from xml.etree import ElementTree as ET @@ -582,10 +581,6 @@ def test_generate_curl_command(self): library_construction_protocol library protocol - - md5 - 506d31c82999a2cbcda138a369955e7d - platform ILLUMINA @@ -599,7 +594,8 @@ def test_generate_curl_command(self): - +