ENH: Support importing MiSeq demultiplexed data

q2_types/per_sample_sequences/__init__.py

@@ -8,9 +8,10 @@
 
 import importlib
 
-from ._format import (CasavaOneEightSingleLanePerSampleDirFmt, FastqGzFormat,
-                      YamlFormat, FastqManifestFormat,
-                      FastqAbsolutePathManifestFormat,
+from ._format import (CasavaOneEightSingleLanePerSampleDirFmt,
+                      CasavaOneEightLanelessPerSampleDirFmt,
+                      FastqGzFormat, YamlFormat,
+                      FastqManifestFormat, FastqAbsolutePathManifestFormat,
                       SingleLanePerSampleSingleEndFastqDirFmt,
                       SingleLanePerSamplePairedEndFastqDirFmt,
                       SingleEndFastqManifestPhred33,
@@ -20,8 +21,9 @@
 from ._type import SequencesWithQuality, PairedEndSequencesWithQuality
 from ._transformer import PerSampleDNAIterators, PerSamplePairedDNAIterators
 
-__all__ = ['CasavaOneEightSingleLanePerSampleDirFmt', 'FastqGzFormat',
-           'YamlFormat', 'FastqManifestFormat',
+__all__ = ['CasavaOneEightSingleLanePerSampleDirFmt',
+           'CasavaOneEightLanelessPerSampleDirFmt',
+           'FastqGzFormat', 'YamlFormat', 'FastqManifestFormat',
            'FastqAbsolutePathManifestFormat',
            'SingleLanePerSampleSingleEndFastqDirFmt',
            'SingleLanePerSamplePairedEndFastqDirFmt', 'SequencesWithQuality',

q2_types/per_sample_sequences/_format.py

@@ -151,10 +151,20 @@ class SingleLanePerSamplePairedEndFastqDirFmt(_SingleLanePerSampleFastqDirFmt):
     pass
 
 
+class CasavaOneEightLanelessPerSampleDirFmt(model.DirectoryFormat):
+    sequences = model.FileCollection(r'.+_.+_R[12]_001\.fastq\.gz',
+                                     format=FastqGzFormat)
+
+    @sequences.set_path_maker
+    def sequences_path_maker(self, sample_id, barcode_id, read_number):
+        return '%s_%s_R%d_001.fastq.gz' % (sample_id, barcode_id, read_number)
+
+
 plugin.register_formats(
     FastqManifestFormat, YamlFormat, FastqGzFormat,
-    CasavaOneEightSingleLanePerSampleDirFmt, _SingleLanePerSampleFastqDirFmt,
-    SingleLanePerSampleSingleEndFastqDirFmt,
+    CasavaOneEightSingleLanePerSampleDirFmt,
+    CasavaOneEightLanelessPerSampleDirFmt,
+    _SingleLanePerSampleFastqDirFmt, SingleLanePerSampleSingleEndFastqDirFmt,
     SingleLanePerSamplePairedEndFastqDirFmt, SingleEndFastqManifestPhred33,
     SingleEndFastqManifestPhred64, PairedEndFastqManifestPhred33,
     PairedEndFastqManifestPhred64

q2_types/per_sample_sequences/_transformer.py

@@ -22,6 +22,7 @@
                FastqAbsolutePathManifestFormat, FastqGzFormat,
                SingleLanePerSamplePairedEndFastqDirFmt, YamlFormat,
                CasavaOneEightSingleLanePerSampleDirFmt,
+               CasavaOneEightLanelessPerSampleDirFmt,
                SingleEndFastqManifestPhred33, SingleEndFastqManifestPhred64,
                PairedEndFastqManifestPhred33, PairedEndFastqManifestPhred64)
 
@@ -76,18 +77,22 @@ def _2(dirfmt: SingleLanePerSamplePairedEndFastqDirFmt) \
     return result
 
 
-def _single_lane_per_sample_fastq_helper(dirfmt, output_cls):
+def _single_lane_per_sample_fastq_helper(dirfmt, output_cls, parse_lane=True):
     result = output_cls()
     manifest = FastqManifestFormat()
     manifest_fh = manifest.open()
     manifest_fh.write('sample-id,filename,direction\n')
     directions = ['forward', 'reverse']
     for path, view in dirfmt.sequences.iter_views(FastqGzFormat):
-
-        sample_id, barcode_id, lane_number, read_number, _ = \
-            str(path).replace('.fastq.gz', '').rsplit('_', maxsplit=4)
+        filename = str(path).replace('.fastq.gz', '')
+        if parse_lane:
+            sample_id, barcode_id, lane_number, read_number, _ = \
+                filename.rsplit('_', maxsplit=4)
+        else:
+            sample_id, barcode_id, read_number, _ = \
+                filename.rsplit('_', maxsplit=3)
         read_number = int(read_number[1:])
-        lane_number = int(lane_number[1:])
+        lane_number = int(lane_number[1:]) if parse_lane else 1
         direction = directions[read_number - 1]
         result.sequences.write_data(view, FastqGzFormat, sample_id=sample_id,
                                     barcode_id=barcode_id,
@@ -119,6 +124,20 @@ def _4(dirfmt: CasavaOneEightSingleLanePerSampleDirFmt) \
         dirfmt, SingleLanePerSamplePairedEndFastqDirFmt)
 
 
+@plugin.register_transformer
+def _10(dirfmt: CasavaOneEightLanelessPerSampleDirFmt) \
+        -> SingleLanePerSampleSingleEndFastqDirFmt:
+    return _single_lane_per_sample_fastq_helper(
+        dirfmt, SingleLanePerSampleSingleEndFastqDirFmt, parse_lane=False)
+
+
+@plugin.register_transformer
+def _11(dirfmt: CasavaOneEightLanelessPerSampleDirFmt) \
+        -> SingleLanePerSamplePairedEndFastqDirFmt:
+    return _single_lane_per_sample_fastq_helper(
+        dirfmt, SingleLanePerSamplePairedEndFastqDirFmt, parse_lane=False)
+
+
 @plugin.register_transformer
 def _5(dirfmt: SingleLanePerSamplePairedEndFastqDirFmt) \
         -> SingleLanePerSampleSingleEndFastqDirFmt:

q2_types/per_sample_sequences/tests/test_format.py

@@ -10,8 +10,10 @@
 import unittest
 
 from q2_types.per_sample_sequences import (
-    CasavaOneEightSingleLanePerSampleDirFmt, FastqGzFormat, YamlFormat,
-    FastqManifestFormat, FastqAbsolutePathManifestFormat,
+    CasavaOneEightSingleLanePerSampleDirFmt,
+    CasavaOneEightLanelessPerSampleDirFmt,
+    FastqGzFormat, YamlFormat, FastqManifestFormat,
+    FastqAbsolutePathManifestFormat,
     SingleEndFastqManifestPhred33, SingleEndFastqManifestPhred64,
     PairedEndFastqManifestPhred33, PairedEndFastqManifestPhred64,
     SingleLanePerSampleSingleEndFastqDirFmt,
@@ -126,6 +128,26 @@ def test_casava_one_eight_slanepsample_dir_fmt_validate_negative(self):
         with self.assertRaisesRegex(ValueError, 'CasavaOneEightSingleLanePer'):
             format.validate()
 
+    def test_miseq_demux_dir_fmt_validate_positive(self):
+        filepath = self.get_data_path('Human-Kneecap_S1_R1_001.fastq.gz')
+        shutil.copy(filepath, self.temp_dir.name)
+
+        format = CasavaOneEightLanelessPerSampleDirFmt(self.temp_dir.name,
+                                                       mode='r')
+
+        format.validate()
+
+    def test_miseq_demux_dir_fmt_validate_negative(self):
+        filepath = self.get_data_path('not-fastq.fastq.gz')
+        shutil.copy(filepath, self.temp_dir.name)
+
+        format = CasavaOneEightLanelessPerSampleDirFmt(self.temp_dir.name,
+                                                       mode='r')
+
+        with self.assertRaisesRegex(ValueError,
+                                    'CasavaOneEightLanelessPerSampleDirFmt'):
+            format.validate()
+
     def test_slanepsample_single_end_fastq_dir_fmt_validate_positive(self):
         filenames = ('single_end_data/MANIFEST', 'metadata.yml',
                      'Human-Kneecap_S1_L001_R1_001.fastq.gz')

q2_types/per_sample_sequences/tests/test_transformer.py

@@ -20,6 +20,7 @@
     SingleLanePerSampleSingleEndFastqDirFmt,
     SingleLanePerSamplePairedEndFastqDirFmt,
     CasavaOneEightSingleLanePerSampleDirFmt,
+    CasavaOneEightLanelessPerSampleDirFmt,
     SingleEndFastqManifestPhred33,
     SingleEndFastqManifestPhred64,
     PairedEndFastqManifestPhred33,
@@ -135,6 +136,44 @@ def test_casava_one_eight_single_lane_per_sample_dirfmt_to_slpspefdf(self):
         for act, exp in zip(obs, input):
             self.assertEqual(act, exp)
 
+    def test_miseq_demux_dirfmt_to_slpssefdf(self):
+        input, obs = self.transform_format(
+            CasavaOneEightLanelessPerSampleDirFmt,
+            SingleLanePerSampleSingleEndFastqDirFmt,
+            filenames=('Human-Kneecap_S1_R1_001.fastq.gz',),
+        )
+
+        input = skbio.io.read(
+            '%s/Human-Kneecap_S1_R1_001.fastq.gz' % str(input),
+            format='fastq', constructor=skbio.DNA
+        )
+        obs = skbio.io.read(
+            '%s/Human-Kneecap_S1_L001_R1_001.fastq.gz' % str(obs),
+            format='fastq', constructor=skbio.DNA
+        )
+
+        for act, exp in zip(obs, input):
+            self.assertEqual(act, exp)
+
+    def test_miseq_demux_dirfmt_to_slpspefdf(self):
+        input, obs = self.transform_format(
+            CasavaOneEightLanelessPerSampleDirFmt,
+            SingleLanePerSamplePairedEndFastqDirFmt,
+            filenames=('Human-Kneecap_S1_R1_001.fastq.gz',),
+        )
+
+        input = skbio.io.read(
+            '%s/Human-Kneecap_S1_R1_001.fastq.gz' % str(input),
+            format='fastq', constructor=skbio.DNA
+        )
+        obs = skbio.io.read(
+            '%s/Human-Kneecap_S1_L001_R1_001.fastq.gz' % str(obs),
+            format='fastq', constructor=skbio.DNA
+        )
+
+        for act, exp in zip(obs, input):
+            self.assertEqual(act, exp)
+
 
 class TestFastqManifestTransformers(TestPluginBase):
     package = "q2_types.per_sample_sequences.tests"