lots of updates to testing and docuementation

.travis.yml

@@ -40,3 +40,5 @@ cache: pip
 script:
   - sh run_jared_tests.sh
   - sh run_andrew_tests.sh
+  - sh run_jh_tests.sh
+

docs/source/PPP_pages/Analyses/vcf_to_sfs.rst

@@ -1,6 +1,6 @@
-================================
+================================================
 vcf_to_sfs.py: Site Frequency Spectrum generator
-================================
+================================================
 .. automodule:: vcf_to_sfs
 
 

docs/source/PPP_pages/input_file_generators.rst

@@ -7,9 +7,9 @@ For several programs that implement analyses of population genomic variation, PP
 .. toctree::
    :maxdepth: 1
 
-   Functions/vcf_format_conversions
-   Functions/vcf_to_ima
-   Functions/vcf_to_treemix
-   Functions/vcf_to_gphocs
-   Functions/vcf_to_fastsimcoal
-   Functions/vcf_to_dadi
+   Input_File_Generators/vcf_format_conversions
+   Input_File_Generators/vcf_to_ima
+   Input_File_Generators/vcf_to_treemix
+   Input_File_Generators/vcf_to_gphocs
+   Input_File_Generators/vcf_to_fastsimcoal
+   Input_File_Generators/vcf_to_dadi

docs/source/PPP_pages/utilities.rst

@@ -9,5 +9,4 @@ The utility functions were developed to perform various tasks often needed when
 
    Utilities/vcf_utilities
    Utilities/bed_utilities
-   Utilities/vcf_BED_to_seq
-
+   Utilities/vcf_bed_to_seq

jhtests/jhtests_run.py

@@ -0,0 +1,222 @@
+import unittest
+import filecmp
+import sys
+import os
+import logging
+import shutil
+import zipfile
+
+## Import scripts to test
+
+import pgpipe.vcf_bed_to_seq as vcf_bed_to_seq
+import pgpipe.vcf_to_sfs as vcf_to_sfs
+import pgpipe.vcf_to_dadi as vcf_to_dadi
+import pgpipe.vcf_to_treemix as vcf_to_treemix
+import pgpipe.vcf_to_gphocs as vcf_to_gphocs
+import pgpipe.vcf_to_fastsimcoal as vcf_to_fastsimcoal
+
+
+## Used to compare two files, returns bool
+def file_comp(test_output, expected_output):
+    return filecmp.cmp(test_output, expected_output)
+
+
+
+class jh_function_tests (unittest.TestCase):
+
+    def test_vcf_bed_to_seq1 (self):
+        vcf_bed_to_seq.run(['--vcf','pan_example.vcf.gz',
+                            '--fasta-reference',"pan_example_ref.fa",
+                            '--model-file',"panmodels.model",
+                            '--modelname',"4Pop",
+                            '--region',"21:4431001-4499000",
+                            '--out','ppp_test_temp.out'])
+
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_bed_to_seq_test.out'))
+
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out')
+
+
+    def test_vcf_to_sfs1 (self):
+
+        vcf_to_sfs.run(['--vcf',"pan_example2.vcf.gz",
+               '--model-file',"panmodels.model",
+               '--modelname','4Pop',
+               '--downsamplesizes','3','3','3','4',
+               '--folded','--outgroup-fasta',"chr22_pan_example2_ref.fa",
+               '--out',"ppp_test_temp.out"])
+
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_to_sfs_test1.txt'))
+
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out')
+
+    def test_vcf_to_sfs2 (self):
+        vcf_to_sfs.run(['--vcf',"pan_example.vcf.gz",
+               '--model-file',"panmodels.model",'--modelname','5Pop',
+               '--downsamplesizes','3','3','3','4','2',
+               '--folded','--outgroup-fasta',"pan_example_ref.fa",
+               '--out',"ppp_test_temp.out"])
+
+
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_to_sfs_test2.txt'))
+
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out')
+
+
+    def test_vcf_to_gphocs1 (self):
+        vcf_to_gphocs.run(['--vcf','pan_example.vcf.gz','--reference',
+                 "pan_example_ref.fa",
+            '--model-file',"panmodels.model",'--modelname',"4Pop",
+            '--bed-file',"pan_example_regions.bed",
+            '--out','ppp_test_temp.out'])
+
+
+
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_gphocs_test.out'))
+
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out')
+    def test_vcf_to_gphocs2 (self):
+        vcf_to_gphocs.run(['--vcf','pan_example2.vcf.gz','--reference',
+                 "chr22_pan_example2_ref.fa",
+            '--model-file',"panmodels.model",'--modelname',"4Pop",
+            '--bed-file',"pan_test_regions.bed",
+            '--out','ppp_test_temp.out','--diploid','False'])
+
+
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_gphocs_test2.out'))
+
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out')      
+
+    def test_vcf_to_treemix1(self):
+
+        vcf_to_treemix.run(['--vcf','pan_example.vcf.gz',
+                        '--model-file',"panmodels.model",
+                        '--modelname',"4Pop",
+                        '--out','ppp_test_temp.out',
+                        '--bed-file',"pan_example_regions.bed",
+                        '--kblock','1000'])
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out.gz',
+                                  'results/vcf_treemixtest1.gz'))
+
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out.gz')
+
+    def test_vcf_to_treemix2(self):
+        vcf_to_treemix.run(['--vcf','pan_example.vcf.gz',
+                        '--model-file',"panmodels.model",
+                        '--modelname',"4Pop",
+                        '--out','ppp_test_temp.out'])
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out.gz',
+                                  'results/vcf_treemixtest2.gz'))
+
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out.gz')        
+
+
+    def test_vcf_to_fastsimcoal1(self):
+
+        vcf_to_fastsimcoal.run(['--vcf',"pan_example.vcf.gz",
+                    '--model-file',"panmodels.model",
+                    '--modelname','3Pop',
+                    '--dim','1','2','m',
+                    '--basename','ppp_test_temp.out'])
+        # must extract archives to check that files match 
+        pz = zipfile.ZipFile('ppp_test_temp.out.zip',mode='r')
+        vz = zipfile.ZipFile('results/vcf_fsc1.zip',mode='r')
+        pznl = pz.namelist()
+        vznl = vz.namelist()
+        assert len(pznl) == len(vznl)
+        for fi in range(len(pznl)):
+            pz.extract(pznl[fi])
+            vz.extract(vznl[fi])
+        # Confirm that the output is what is expected
+            self.assertTrue(file_comp(pznl[fi],vznl[fi]))
+        # remove extracted files 
+            self.addCleanup(os.remove, pznl[fi])
+            self.addCleanup(os.remove, vznl[fi])          
+
+
+    def test_vcf_to_fastsimcoal2(self):
+
+        vcf_to_fastsimcoal.run(['--vcf',"pan_example2.vcf.gz",
+               '--model-file',"panmodels.model",
+               '--modelname','5Pop',
+               '--downsamplesizes','3','3','3','4','2',
+               '--basename','ppp_test_temp.out',
+               '--folded','--dim','1','2','m',
+               '--outgroup-fasta',"chr22_pan_example2_ref.fa"])
+        # must extract archives to check that files match 
+        pz = zipfile.ZipFile('ppp_test_temp.out.zip',mode='r')
+        vz = zipfile.ZipFile('results/vcf_fsc2.zip',mode='r')
+        pznl = pz.namelist()
+        vznl = vz.namelist()
+        assert len(pznl) == len(vznl)
+        for fi in range(len(pznl)):
+            pz.extract(pznl[fi])
+            vz.extract(vznl[fi])
+        # Confirm that the output is what is expected
+            self.assertTrue(file_comp(pznl[fi],vznl[fi]))
+        # remove extracted files 
+            self.addCleanup(os.remove, pznl[fi])
+            self.addCleanup(os.remove, vznl[fi])  
+
+
+    def test_vcf_to_dadi1(self):
+
+        vcf_to_dadi.run(['--vcf','pan_example.vcf.gz',
+                '--model-file',"panmodels.model",
+                '--modelname',"4Pop",'--out','ppp_test_temp.out',
+                '--comment','testing bedfile',
+                '--bed-file',"pan_example_regions.bed"])
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_dadisnp_bedfile_test.out'))
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out')
+
+
+    def test_vcf_to_dadi2(self):                        
+        vcf_to_dadi.run(['--vcf',"pan_example2.vcf.gz",
+                '--model-file',"panmodels.model",
+                '--modelname',"4Pop",'--out','ppp_test_temp.out',
+                '--comment','testing comment'])
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_dadisnp_test.out'))
+        # Remove the ouput
+        self.addCleanup(os.remove, 'ppp_test_temp.out')
+
+    def test_vcf_to_dadi3(self):                        
+        vcf_to_dadi.run(['--vcf',"pan_example2.vcf.gz",'--model-file',
+                 "panmodels.model",
+                 '--modelname',"4Pop",'--out','ppp_test_temp.out',
+                 '--comment','testing outgroup-fasta','--outgroup-fasta',
+                 "chr22_pan_example2_ref.fa"])
+        # Confirm that the output is what is expected
+        self.assertTrue(file_comp('ppp_test_temp.out',
+                                  'results/vcf_dadisnp_fasta_test.out'))
+        # Remove the ouput 
+        self.addCleanup(os.remove, 'ppp_test_temp.out')          
+
+if __name__ == "__main__":
+    unittest.main(verbosity = 2)
+
+
+

jhtests/pan_test_regions.bed

@@ -1,2 +1,3 @@
-22	19999999	20001000
-22	29999999	30010000
+22	14430599	14431600
+22	15658547	15659548
+