From 04d7af0d15c28ac5bc4a3a54ea499fb9de0290b8 Mon Sep 17 00:00:00 2001 From: Charles Cowart <42684307+charles-cowart@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:30:00 -0700 Subject: [PATCH] latest updates based on testing (#137) * latest updates based on testing * updated test parameters --- sequence_processing_pipeline/NuQCJob.py | 12 ++++++---- .../tests/test_NuQCJob.py | 22 +++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/sequence_processing_pipeline/NuQCJob.py b/sequence_processing_pipeline/NuQCJob.py index 7d28fff..74c7705 100644 --- a/sequence_processing_pipeline/NuQCJob.py +++ b/sequence_processing_pipeline/NuQCJob.py @@ -83,7 +83,8 @@ def __init__(self, fastq_root_dir, output_path, sample_sheet_path, self.queue_name = queue_name self.node_count = node_count self.wall_time_limit = wall_time_limit - self.jmem = jmem + # raise an Error if jmem is not a valid floating point value. + self.jmem = float(jmem) self.fastp_path = fastp_path self.minimap2_path = minimap2_path self.samtools_path = samtools_path @@ -408,15 +409,18 @@ def _generate_job_script(self, max_bucket_size): else: gres_value = 2 + # As slurm expects memory sizes to be integer values, convert results + # to int to drop floating point component before passing to jinja2 as + # a string. if max_bucket_size < gigabyte: mod_wall_time_limit = self.wall_time_limit - mod_jmem = self.jmem + mod_jmem = str(int(self.jmem)) elif max_bucket_size < (2 * gigabyte): mod_wall_time_limit = self.wall_time_limit * 1.5 - mod_jmem = self.jmem * 4.5 + mod_jmem = str(int(self.jmem * 4.5)) else: mod_wall_time_limit = self.wall_time_limit * 2 - mod_jmem = self.jmem * 7.5 + mod_jmem = str(int(self.jmem * 7.5)) job_script_path = join(self.output_path, 'process_all_fastq_files.sh') template = self.jinja_env.get_template("nuqc_job.sh") diff --git a/sequence_processing_pipeline/tests/test_NuQCJob.py b/sequence_processing_pipeline/tests/test_NuQCJob.py index ed5ec63..37c0d07 100644 --- a/sequence_processing_pipeline/tests/test_NuQCJob.py +++ b/sequence_processing_pipeline/tests/test_NuQCJob.py @@ -591,7 +591,7 @@ def test_nuqcjob_creation(self): with self.assertRaises(PipelineError) as e: NuQCJob(self.fastq_root_path, self.output_path, 'not/path/to/sample/sheet', self.mmi_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -606,7 +606,7 @@ def test_nuqcjob_creation(self): nuqcjob = NuQCJob(self.fastq_root_path, self.output_path, self.tmp_file_path, self.mmi_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -624,7 +624,7 @@ def test_nuqcjob_creation(self): "sample-sheet."): NuQCJob(self.fastq_root_path, self.output_path, self.tmp_file_path, self.mmi_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -632,14 +632,14 @@ def test_nuqcjob_creation(self): " for HumanFiltering"): NuQCJob(self.fastq_root_path, self.output_path, self.bad_sheet_bools_path, self.mmi_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') def test_error_msg_from_logs(self): job = NuQCJob(self.fastq_root_path, self.output_path, self.good_sample_sheet_path, self.mmi_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -668,14 +668,14 @@ def test_assay_value(self): " valid sample-sheet."): NuQCJob(self.fastq_root_path, self.output_path, self.bad_sample_sheet_path, self.mmi_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') def test_audit(self): job = NuQCJob(self.fastq_root_path, self.output_path, self.good_sample_sheet_path, self.mmi_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -1105,7 +1105,7 @@ def test_completed_file_generation(self): job = NuQCJob(self.fastq_root_path, self.output_path, self.good_sample_sheet_path, double_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -1128,7 +1128,7 @@ def test_completed_file_generation_some_failures(self): job = NuQCJob(self.fastq_root_path, self.output_path, self.good_sample_sheet_path, double_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -1140,7 +1140,7 @@ def test_generate_job_script(self): double_db_paths = ["db_path/mmi_1.db", "db_path/mmi_2.db"] job = NuQCJob(self.fastq_root_path, self.output_path, self.good_sample_sheet_path, double_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '') @@ -1153,7 +1153,7 @@ def test_regular_expressions(self): double_db_paths = ["db_path/mmi_1.db", "db_path/mmi_2.db"] job = NuQCJob(self.fastq_root_path, self.output_path, self.good_sample_sheet_path, double_db_paths, - 'queue_name', 1, 1440, '8gb', + 'queue_name', 1, 1440, '8', 'fastp', 'minimap2', 'samtools', [], self.qiita_job_id, 1000, '')