Merge pull request #114 from supermaxiste/logfix

Log and benchmark fixes
supermaxiste · Aug 17, 2023 · e878596 · e878596
2 parents a1386bd + a9eb1ce
commit e878596
Show file tree

Hide file tree

Showing 10 changed files with 153 additions and 154 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -6,7 +6,6 @@ name: ARPEGGIO_run
 # events but only for the master branch. Added trigger to publishhed and created 
 # releases as well
 on:
-  push:
   pull_request:
     branches: [ master ]
   release:

diff --git a/rules/alignment.smk b/rules/alignment.smk
@@ -12,17 +12,17 @@ rule bismark_prepare_genome_1:
     output:
         control=f"{GENOME_1}Bisulfite_Genome/CT_conversion/genome_mfa.CT_conversion.fa",
     log:
-        f"logs/bismark_prepare_genome_1.log",
+        f"{OUTPUT_DIR}logs/bismark_prepare_genome_1.log",
     params:
         genome1=lambda w, output: os.path.split(
             os.path.split(os.path.split(output.control)[0])[0]
         )[0],
     benchmark:
-        f"{OUTPUT_DIR}benchmark/prepare_genome.txt"
+        f"{OUTPUT_DIR}benchmark/prepare_genome_1.txt"
     conda:
         "../envs/environment.yaml"
     shell:
-        "bismark_genome_preparation {params.genome1}"
+        "bismark_genome_preparation {params.genome1} 2> {log}"
 
 
 ## Run Bismark to prepare synthetic bisulfite converted genomes
@@ -32,17 +32,17 @@ rule bismark_prepare_genome_2:
     output:
         control=f"{GENOME_2}Bisulfite_Genome/CT_conversion/genome_mfa.CT_conversion.fa",
     log:
-        f"logs/bismark_prepare_genome_2.log",
+        f"{OUTPUT_DIR}logs/bismark_prepare_genome_2.log",
     params:
         genome2=lambda w, output: os.path.split(
             os.path.split(os.path.split(output.control)[0])[0]
         )[0],
     benchmark:
-        f"{OUTPUT_DIR}benchmark/prepare_genome.txt"
+        f"{OUTPUT_DIR}benchmark/prepare_genome_2.txt"
     conda:
         "../envs/environment.yaml"
     shell:
-        "bismark_genome_preparation {params.genome2}"
+        "bismark_genome_preparation {params.genome2} 2> {log}"
 
 
 ## Run Bismark to perform alignment to the first parental genome (GENOME_PARENT_1) if reads are single-end.
@@ -62,7 +62,7 @@ rule bismark_alignment_SE_1:
         if config["RUN_TRIMMING"]
         else f"{OUTPUT_DIR}Bismark/{{sample}}_1/1.{{sample}}_bismark_bt2_SE_report.txt",
     log:
-        f"logs/bismark_alignment_{{sample}}_SE_1.log",
+        f"{OUTPUT_DIR}logs/bismark_alignment_{{sample}}_SE_1.log",
     params:
         output=lambda w, output: os.path.split(output.sample)[0],
         genome1=lambda w, input: os.path.split(
@@ -76,7 +76,7 @@ rule bismark_alignment_SE_1:
         "../envs/environment.yaml"
     threads: CORES
     shell:
-        "bismark --prefix {params.prefix} --multicore {params.bismark_cores}  -o {params.output} --temp_dir {params.output} --genome {params.genome1} {input.fastq}"
+        "bismark --prefix {params.prefix} --multicore {params.bismark_cores}  -o {params.output} --temp_dir {params.output} --genome {params.genome1} {input.fastq} 2> {log}"
 
 
 ## Run Bismark to perform alignment to the second parental genome (GENOME_PARENT_2) if reads are single-end.
@@ -96,7 +96,7 @@ rule bismark_alignment_SE_2:
         if config["RUN_TRIMMING"]
         else f"{OUTPUT_DIR}Bismark/{{sample}}_2/2.{{sample}}_bismark_bt2_SE_report.txt",
     log:
-        f"logs/bismark_alignment_{{sample}}_SE_2.log",
+        f"{OUTPUT_DIR}logs/bismark_alignment_{{sample}}_SE_2.log",
     params:
         output=lambda w, output: os.path.split(output.sample)[0],
         genome2=lambda w, input: os.path.split(
@@ -110,7 +110,7 @@ rule bismark_alignment_SE_2:
         "../envs/environment.yaml"
     threads: CORES
     shell:
-        "bismark --prefix {params.prefix} --multicore {params.bismark_cores}  -o {params.output} --temp_dir {params.output} --genome {params.genome2} {input.fastq}"
+        "bismark --prefix {params.prefix} --multicore {params.bismark_cores}  -o {params.output} --temp_dir {params.output} --genome {params.genome2} {input.fastq} 2> {log}"
 
 
 ## Run Bismark to perform alignment to the first parental genome (GENOME_PARENT_1) if reads are paired-end.
@@ -133,7 +133,7 @@ rule bismark_alignment_PE_1:
         if config["RUN_TRIMMING"]
         else f"{OUTPUT_DIR}Bismark/{{sample}}_1/1.{{sample}}_{str(config['PAIR_1'])}_bismark_bt2_PE_report.txt",
     log:
-        f"logs/bismark_alignment_{{sample}}_PE_1.log",
+        f"{OUTPUT_DIR}logs/bismark_alignment_{{sample}}_PE_1.log",
     params:
         output=lambda w, output: os.path.split(output.sample)[0],
         genome1=lambda w, input: os.path.split(
@@ -147,7 +147,7 @@ rule bismark_alignment_PE_1:
         "../envs/environment.yaml"
     threads: CORES
     shell:
-        "bismark --prefix {params.prefix} --multicore {params.bismark_cores} --genome {params.genome1} -1 {input.fastq1} -2 {input.fastq2} -o {params.output} --temp_dir {params.output}"
+        "bismark --prefix {params.prefix} --multicore {params.bismark_cores} --genome {params.genome1} -1 {input.fastq1} -2 {input.fastq2} -o {params.output} --temp_dir {params.output} 2> {log}"
 
 
 ## Run Bismark to perform alignment to the second parental genome (GENOME_PARENT_2) if reads are paired-end.
@@ -170,7 +170,7 @@ rule bismark_alignment_PE_2:
         if config["RUN_TRIMMING"]
         else f"{OUTPUT_DIR}Bismark/{{sample}}_2/2.{{sample}}_{str(config['PAIR_1'])}_bismark_bt2_PE_report.txt",
     log:
-        f"logs/bismark_alignment_{{sample}}_PE_2.log",
+        f"{OUTPUT_DIR}logs/bismark_alignment_{{sample}}_PE_2.log",
     params:
         output=lambda w, output: os.path.split(output.sample)[0],
         genome2=lambda w, input: os.path.split(
@@ -184,4 +184,4 @@ rule bismark_alignment_PE_2:
         "../envs/environment.yaml"
     threads: CORES
     shell:
-        "bismark --prefix {params.prefix} --multicore {params.bismark_cores} --genome {params.genome2} -1 {input.fastq1} -2 {input.fastq2} -o {params.output} --temp_dir {params.output}"
+        "bismark --prefix {params.prefix} --multicore {params.bismark_cores} --genome {params.genome2} -1 {input.fastq1} -2 {input.fastq2} -o {params.output} --temp_dir {params.output} 2> {log}"
diff --git a/rules/build_eagle.smk b/rules/build_eagle.smk
@@ -44,11 +44,11 @@ rule download_eagle:
     params:
         eagle_version=EAGLE_VERSION,
     log:
-        "logs/download_eagle.log",
+        f"{OUTPUT_DIR}logs/download_eagle.log",
     conda:
         ENV_PATH
     shell:
-        "wget https://github.com/tony-kuo/eagle/archive/v{params.eagle_version}.tar.gz -O {output.eagle_tar}"
+        "wget https://github.com/tony-kuo/eagle/archive/v{params.eagle_version}.tar.gz -O {output.eagle_tar} 2> {log}"
 
 
 rule extract_eagle:
@@ -59,11 +59,11 @@ rule extract_eagle:
     params:
         build_prefix=lambda w, input: os.path.split(input.eagle_tar)[0],
     log:
-        "logs/extract_eagle.log",
+        f"{OUTPUT_DIR}logs/extract_eagle.log",
     conda:
         ENV_PATH
     shell:
-        "tar -C {params.build_prefix} -vxf {input.eagle_tar}"
+        "tar -C {params.build_prefix} -vxf {input.eagle_tar} 2> {log}"
 
 
 rule download_htslib:
@@ -73,11 +73,11 @@ rule download_htslib:
         htslib_version=HTSLIB_VERSION,
         htslib_tar_name=HTSLIB_TAR_NAME,
     log:
-        "logs/download_htslib.log",
+        f"{OUTPUT_DIR}logs/download_htslib.log",
     conda:
         ENV_PATH
     shell:
-        "wget https://github.com/samtools/htslib/releases/download/{params.htslib_version}/{params.htslib_tar_name} -O {output.htslib_tar}"
+        "wget https://github.com/samtools/htslib/releases/download/{params.htslib_version}/{params.htslib_tar_name} -O {output.htslib_tar} 2> {log}"
 
 
 rule extract_htslib:
@@ -88,11 +88,11 @@ rule extract_htslib:
     params:
         build_prefix=lambda w, input: os.path.split(input.htslib_tar)[0],
     log:
-        "logs/extract_htslib.log",
+        f"{OUTPUT_DIR}logs/extract_htslib.log",
     conda:
         ENV_PATH
     shell:
-        "tar -C {params.build_prefix} -vxf {input.htslib_tar}"
+        "tar -C {params.build_prefix} -vxf {input.htslib_tar} 2> {log}"
 
 
 rule build_eagle:
@@ -106,12 +106,12 @@ rule build_eagle:
         eagle_dir_path=lambda w, input: os.path.split(input.eagle_mk)[0],
         eagle_mk_flags=EAGLE_MK_FLAGS,
     log:
-        "logs/build_eagle.log",
+        f"{OUTPUT_DIR}logs/build_eagle.log",
     conda:
         ENV_PATH
     threads: CORES
     shell:
-        "{params.eagle_mk_env} make -j {threads} -C {params.eagle_dir_path} {params.eagle_mk_flags}"
+        "{params.eagle_mk_env} make -j {threads} -C {params.eagle_dir_path} {params.eagle_mk_flags} 2> {log}"
 
 
 rule install_eagle:
@@ -124,8 +124,8 @@ rule install_eagle:
         eagle_dir_path=lambda w, input: os.path.split(input.eagle_bin)[0],
         eagle_mk_flags=EAGLE_MK_FLAGS,
     log:
-        "logs/install_eagle.log",
+        f"{OUTPUT_DIR}logs/install_eagle.log",
     conda:
         ENV_PATH
     shell:
-        "{params.eagle_mk_env} make -C {params.eagle_dir_path} install {params.eagle_mk_flags}"
+        "{params.eagle_mk_env} make -C {params.eagle_dir_path} install {params.eagle_mk_flags} 2> {log}"
diff --git a/rules/deduplication.smk b/rules/deduplication.smk
@@ -18,15 +18,15 @@ rule deduplication_SE:
         if config["RUN_TRIMMING"]
         else f"{OUTPUT_DIR}Bismark/deduplication/{{sample}}_{{one_or_two}}/{{one_or_two}}.{{sample}}_bismark_bt2.deduplicated.bam",
     log:
-        f"logs/deduplication_{{sample}}_{{one_or_two}}_SE.log",
+        f"{OUTPUT_DIR}logs/deduplication_{{sample}}_{{one_or_two}}_SE.log",
     params:
         f"{OUTPUT_DIR}Bismark/deduplication/{{sample}}_{{one_or_two}}/",
     benchmark:
         f"{OUTPUT_DIR}benchmark/dedup_se{{one_or_two}}_{{sample}}.txt"
     conda:
         "../envs/environment.yaml"
     shell:
-        "deduplicate_bismark -s --output_dir {params} --bam {input}"
+        "deduplicate_bismark -s --output_dir {params} --bam {input} 2> {log}"
 
 
 ## Run deduplication of the alignments to remove duplicated reads for PE reads (GENOME_PARENT_1)
@@ -42,12 +42,12 @@ rule deduplication_PE:
         if config["RUN_TRIMMING"]
         else f"{OUTPUT_DIR}Bismark/deduplication/{{sample}}_{{one_or_two}}/{{one_or_two}}.{{sample}}_{str(config['PAIR_1'])}_bismark_bt2_pe.deduplicated.bam",
     log:
-        f"logs/deduplication_{{sample}}_{{one_or_two}}_PE.log",
+        f"{OUTPUT_DIR}logs/deduplication_{{sample}}_{{one_or_two}}_PE.log",
     params:
         f"{OUTPUT_DIR}Bismark/deduplication/{{sample}}_{{one_or_two}}/",
     benchmark:
         f"{OUTPUT_DIR}benchmark/dedup_pe{{one_or_two}}_{{sample}}.txt"
     conda:
         "../envs/environment.yaml"
     shell:
-        "deduplicate_bismark -p --output_dir {params} --bam {input}"
+        "deduplicate_bismark -p --output_dir {params} --bam {input} 2> {log}"