Set default Mappingquality filter to 30, bowtie2 option to fast. reworked tests for new defaults

Author: Johannes Groos

vxdetector/VXdetector.py

@@ -4,17 +4,11 @@
 import glob
 import os
 import sys
-import time
-
-# SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-# sys.path.append(os.path.dirname(SCRIPT_DIR))
-# sys.path.append('$CONDA/lib/python3.9/site-packages')
-
-import pandas as pd  # noqa: E402
-from vxdetector.interact_bowtie2 import mapbowtie2, buildbowtie2  # noqa: E402
-from vxdetector.interact_bedtools import overlap  # noqa: E402
-import vxdetector.Output_counter as Output_counter  # noqa: E402
-import vxdetector.files_manager as files_manager  # noqa: E402
+import pandas as pd
+import vxdetector.Output_counter as Output_counter
+import vxdetector.files_manager as files_manager
+from vxdetector.interact_bowtie2 import mapbowtie2, buildbowtie2
+from vxdetector.interact_bedtools import overlap
 
 
 def do_statistic(result):
@@ -252,8 +246,4 @@ def main():
 
 
 if __name__ == '__main__':
-    start_time = time.time()
     main()
-    print("--- %s seconds ---" % (time.time() - start_time))
-    with open('/homes/jgroos/Desktop/Output/time.txt', 'a') as f:
-        f.write("\n --- %s seconds ---" % (time.time() - start_time))

vxdetector/tests/test_Output_counter.py

@@ -2,12 +2,7 @@
 
 import unittest
 import os
-import sys
-
-import_path = f'{__file__.rsplit("/", 2)[0]}/'
-sys.path.append(import_path)
-
-import Output_counter as oc  # noqa: E402
+import vxdetector.Output_counter as oc
 
 
 data_path = f'{os.path.dirname(__file__)}/test_data/'
@@ -79,7 +74,7 @@ def test_no_aligned(self):
         row = {'Number of Reads': 64421,
                'Unaligned Reads [%]': 100,
                'Not properly paired': 'not paired',
-               'Sequenced variable region': 'No variable Region',
+               'Sequenced variable region': 'Not 16S',
                'V1': 0.0, 'V2': 0.0,
                'V3': 0.0, 'V4': 0.0, 'V5': 0.0,
                'V6': 0.0, 'V7': 0.0, 'V8': 0.0, 'V9': 0.0,
@@ -106,13 +101,15 @@ def test_return_value(self):
         new_row = oc.create_row(test_data, paired=False)
         self.assertEqual(new_row, row)
         test_data = f'{data_path}paired/'
-        row = {'Not aligned to a variable region': 0.014914649681528664,
-               'Not properly paired': 'not paired', 'Number of Reads': 64421,
-               'Sequenced variable region': 'V45',
-               'Unaligned Reads [%]': 12.189999999999998, 'V1': 0.0,
-               'V2': 0.0, 'V3': 0.0, 'V4': 48.9871668789809,
-               'V5': 38.800461146496815, 'V6': 0.007457324840764332,
-               'V7': 0.0, 'V8': 0.0, 'V9': 0.0}
+        row = {'Number of Reads': 64421,
+               'Unaligned Reads [%]': 12.189999999999998,
+               'Not properly paired': 'not paired',
+               'Sequenced variable region': 'V45', 'V1': 0.0,
+               'V2': 0.0, 'V3': 0.0, 'V4': 45.375854271356786,
+               'V5': 42.3973743718593, 'V6': 0.0122571189279732,
+               'V7': 0.0, 'V8': 0.0, 'V9': 0.0,
+               'Not aligned to a variable region': 0.0245142378559464}
+
         new_row = oc.create_row(test_data, paired=False)
         self.assertEqual(new_row, row)
 

vxdetector/tests/test_VXdetector.py

@@ -6,29 +6,22 @@
 import os
 import sys
 from glob import glob
-
-
-sys.path.append('$CONDA/lib/python3.9/site-packages')
-
-import pandas as pd  # noqa: E402
-
-sys.path.append(f'{__file__.rsplit("/", 2)[0]}/')
-
-import VXdetector as vx  # noqa: E402
-import shutil  # noqa: E402
+import pandas as pd
+import vxdetector.VXdetector as vx
+import shutil
 
 path = f'{os.path.dirname(__file__)}/'
 output_test = f'{path}test_data/Output_test.csv'
 result = {'5011_S225_L001': {'Number of Reads': 64421,
                              'Unaligned Reads [%]': 12.189999999999998,
-                             'Not properly paired': 0.005588239859673088,
+                             'Not properly paired': 0.004843141211716676,
                              'Sequenced variable region': 'V45', 'V1': 0.0,
-                             'V2': 0.0, 'V3': 0.0, 'V4': 48.9871668789809,
-                             'V5': 38.800461146496815,
-                             'V6': 0.007457324840764332,
+                             'V2': 0.0, 'V3': 0.0, 'V4': 45.375854271356786,
+                             'V5': 42.3973743718593,
+                             'V6': 0.0122571189279732,
                              'V7': 0.0, 'V8': 0.0, 'V9': 0.0,
                              'Not aligned to a variable region':
-                             0.014914649681528664}}
+                             0.0245142378559464}}
 
 
 class test_do_output(unittest.TestCase):
@@ -53,19 +46,20 @@ def test_csv_output(self):
         self.assertEqual(output, content)
 
     def test_numeric_conversion(self):
-        mixed_result = {'5011_S225_L001': {'Number of Reads': '64421',
+        mixed_result = {'5011_S225_L001': {'Not aligned to a variable region':
+                                           0.0245142378559464,
+                                           'Number of Reads': '64421',
                                            'Unaligned Reads [%]':
                                            '12.189999999999998',
+                                           'V7': 0.0, 'V8': 0.0, 'V9': 0.0,
                                            'Not properly paired':
-                                           0.005588239859673088,
+                                           0.004843141211716676,
                                            'Sequenced variable region':
                                            'V45', 'V1': 0.0, 'V2': 0.0,
-                                           'V3': 0.0, 'V4': '48.9871668789809',
-                                           'V5': 38.800461146496815,
-                                           'V6': 0.007457324840764332,
-                                           'V7': 0.0, 'V8': 0.0, 'V9': 0.0,
-                                           'Not aligned to a variable region':
-                                           0.014914649681528664}}
+                                           'V3': 0.0, 'V4': 45.375854271356786,
+                                           'V5': '42.3973743718593',
+                                           'V6': 0.0122571189279732,
+                                           }}
         single_file = True
         new_file = f'{self.fp_tmpdir}test3.csv'
         vx.do_output(mixed_result, new_file, single_file)
@@ -84,9 +78,9 @@ def test_output_options(self):
                     'paired,Sequenced variable region,V1,V2,V3,V4,V5,V6,'
                     'V7,V8,V9,Not aligned to a variable region\n'
                     '5011_S225_L001,64421,12.189999999999998,'
-                    '0.005588239859673088,V45,0.0,0.0,0.0,48.9871668789809,'
-                    '38.800461146496815,0.007457324840764332,0.0,0.0,0.0'
-                    ',0.014914649681528664\n')
+                    '0.004843141211716676,V45,0.0,0.0,0.0,45.375854271356786,'
+                    '42.3973743718593,0.0122571189279732,0.0,0.0,0.0'
+                    ',0.0245142378559464\n')
         capturedOutput = io.StringIO()
         sys.stdout = capturedOutput
         new_file = sys.stdout
@@ -100,6 +94,12 @@ def test_output_options(self):
 
 
 class test_do_statistic(unittest.TestCase):
+    def setUp(self):
+        self.fp_tmpdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.fp_tmpdir)
+
     def test_order(self):
         result = pd.read_csv(f'{path}test_data/mixed.csv', index_col=0)
         columns = ''.join(['Number of Reads',
@@ -120,7 +120,7 @@ def test_basic_function(self):
         result = pd.read_csv(f'{path}test_data/result_unpaired.csv',
                              index_col=0)
         statistic = vx.do_statistic(result).round(5)
-        statistic.to_csv(f'{path}test_data/statistic1.csv')
+        statistic.to_csv(f'{self.fp_tmpdir}/statistic1.csv')
         self.assertTrue(statistic.equals(expected))
 
 
@@ -133,9 +133,14 @@ def tearDown(self):
                           'Output/test_data.csv'):
             os.remove(f'{__file__.rsplit("/", 3)[0]}/Output/test_data.csv')
         file_list = glob(f'{__file__.rsplit("/", 3)[0]}/Indexed_bt2/*.bt2')
+        directory_list = glob(f'{__file__.rsplit("/", 3)[0]}/tmp_files_*')
+        if os.path.exists(f'{path}test_data/dir_test_actual.csv'):
+            file_list.append(f'{path}test_data/dir_test_actual.csv')
         for file in file_list:
             os.remove(file)
         shutil.rmtree(self.fp_tmpdir)
+        for directory in directory_list:
+            shutil.rmtree(directory)
 
     def test_singleFile(self):
         expected = f'{path}test_data/Output_test.csv'

vxdetector/tests/test_data/Output_test.csv

@@ -1,2 +1,2 @@
 ,Number of Reads,Unaligned Reads [%],Not properly paired,Sequenced variable region,V1,V2,V3,V4,V5,V6,V7,V8,V9,Not aligned to a variable region
-5011_S225_L001,64421,12.189999999999998,0.005588239859673088,V45,0.0,0.0,0.0,48.9871668789809,38.800461146496815,0.007457324840764332,0.0,0.0,0.0,0.014914649681528664
+5011_S225_L001,64421,12.189999999999998,0.004843141211716676,V45,0.0,0.0,0.0,45.375854271356786,42.3973743718593,0.0122571189279732,0.0,0.0,0.0,0.0245142378559464

vxdetector/tests/test_data/dir_test.csv

@@ -1,8 +1,10 @@
 ,Number of Reads,Unaligned Reads [%],Not properly paired,Sequenced variable region,V1,V2,V3,V4,V5,V6,V7,V8,V9,Not aligned to a variable region
-Average,43829.8,11.374,0.004813250141643036,V45,0.0,0.0,0.0,62.51489118968541,25.916430839388727,0.0,0.0,0.0,0.0,0.19467797092586253
-Standard deviation,7958.908825963519,2.140789573965645,0.0010836849387105916,,0.0,0.0,0.0,16.35871651979062,14.321883965171105,0.0,0.0,0.0,0.0,0.15121699956111073
-5001_S229_L001,48981.0,8.64,0.0048181948102325395,V4,0.0,0.0,0.0,76.6020787818603,14.697437934458788,0.0,0.0,0.0,0.0,0.06048328368090036
-5002_S245_L001,52063.0,9.659999999999997,0.00526285461844304,V4,0.0,0.0,0.0,78.8072340425532,11.258176291793314,0.0,0.0,0.0,0.0,0.27458966565349546
-5004_S20_L001,31352.0,11.920000000000002,0.006379178361826996,V45,0.0,0.0,0.0,66.46267564966314,21.53255052935515,0.0,0.0,0.0,0.0,0.08477382098171318
-5005_S265_L001,42162.0,13.260000000000005,0.0037711683506475024,V45,0.0,0.0,0.0,44.40946455889852,42.19783783783784,0.0,0.0,0.0,0.0,0.132697603263641
-5006_S277_L001,44591.0,13.39,0.0038348545670651027,V45,0.0,0.0,0.0,46.2930029154519,39.896151603498545,0.0,0.0,0.0,0.0,0.4208454810495627
+Average,31314.428571428572,8.124285714285715,not paired,V45,0.0,0.0,0.0,39.54846684877456,23.396177364786663,0.0,0.0,0.0,0.0,0.35964150072449014
+Standard deviation,22340.102408427934,5.818693602355708,,,0.0,0.0,0.0,29.026142068327204,18.3249976982372,0.0,0.0,0.0,0.0,0.5215716865831309
+5001_S229_L001,48981.0,8.64,0.0038790551438312817,V4,0.0,0.0,0.0,75.65584204413473,15.598048780487806,0.0,0.0,0.0,0.0,0.10610917537746806
+5002_S245_L001,52063.0,9.659999999999997,0.00447534717553733,V45,0.0,0.0,0.0,56.54221176470588,32.30983529411765,0.0,0.0,0.0,0.0,1.4879529411764707
+5004_S20_L001,31352.0,11.920000000000002,0.004880071446797652,V45,0.0,0.0,0.0,56.73990697674419,31.13525581395349,0.0,0.0,0.0,0.0,0.20483720930232557
+5005_S265_L001,42162.0,13.260000000000005,0.0020160333950002374,V45,0.0,0.0,0.0,43.72145867098865,42.73737439222042,0.0,0.0,0.0,0.0,0.2811669367909238
+5006_S277_L001,44591.0,13.39,0.0027359781121751026,V45,0.0,0.0,0.0,44.17984848484848,41.99272727272727,0.0,0.0,0.0,0.0,0.43742424242424244
+no_qual_paired,26.0,0.0,0.0,Not 16S,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+no_qual_test,26.0,0.0,not paired,Not 16S,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0