Update and refine conda envs, to improve stability and solvability.

In particular, the picard and the qualimap envs were long standing
issues, as they did not get solved properly by conda - mamba was needed.
This hopefully helps with #11 and #25 as well.
We updated envs according to the suggestions of @akriese, with the
exception of the vep env, for which we could not easily get the bcftools
version updated without breaking VEP completely.

envs/bcftools.yaml

@@ -2,7 +2,7 @@ channels:
   - bioconda
   - conda-forge
 dependencies:
-  - bcftools ==1.13
+  - bcftools ==1.16
   # Needed for bcftools stats
   - tectonic ==0.9.0
   # - texlive-core =20210325

envs/bowtie2.yaml

@@ -9,5 +9,5 @@ dependencies:
   - numpy ==1.21.2
   # Tools
   - bowtie2 ==2.4.1
-  - samtools ==1.12
+  - samtools ==1.16.1
   - tbb ==2020.2

envs/bwa-mem2.yaml

@@ -8,5 +8,5 @@ dependencies:
   - numpy ==1.21.2
   # Tools
   - bwa-mem2 ==2.2.1
-  - picard ==2.20.1
-  - samtools ==1.12
+  - picard ==2.27.4
+  - samtools ==1.16.1

envs/bwa.yaml

@@ -8,5 +8,5 @@ dependencies:
   - numpy ==1.21.2
   # Tools
   - bwa ==0.7.17
-  - samtools ==1.12
-  - picard ==2.20.1
+  - samtools ==1.16.1
+  - picard ==2.27.4

envs/freebayes.yaml

@@ -11,7 +11,7 @@ dependencies:
   - numpy ==1.21.2
   # Tools
   - freebayes =1.3.1
-  - bcftools =1.11
+  - bcftools ==1.16
   - parallel =20190522
   - bedtools >=2.29
   - sed =4.7

envs/gatk.yaml

@@ -1,6 +1,7 @@
 channels:
-  - bioconda
+  # conda-forge channel first, as required by issue #25
   - conda-forge
+  - bioconda
   - r
 dependencies:
   # Python

envs/hafpipe.yaml

@@ -15,8 +15,8 @@ dependencies:
   # We need samtools for indexing bam files.
   # Also, tabix and bgzip are dependencies of hafpipe,
   # which both seem to be provided via samtools as well.
-  - samtools ==1.12
-  # - htslib ==1.10
+  - samtools ==1.16.1
+  # - htslib ==1.16
   # - tabix ==1.10
 
   # hafpipe needs R version >= 3.2, so we use the same that

envs/multiqc.yaml

@@ -1,4 +1,6 @@
 channels:
+  # Use anaconda as first channel to fix issue #25
+  - anaconda
   - bioconda
   - conda-forge
 dependencies:

envs/picard.yaml

@@ -1,12 +1,25 @@
 channels:
-  - bioconda
+  # This env was a long term problem, as it failed to get resolved by conda (working perfectly fine
+  # with mamba though). We now tried to use conda-forge as the first channel, and have updated a few
+  # versions of the packages below. It seems to be working... for now.
+  # conda-forge channel first, as required by issue #25
   - conda-forge
+  - bioconda
+  # In issue #11 it was mentioned that removing some channels solves the problem.
+  # But of course it did not... trying to add this one again, in the hope that this
+  # helps with this nightmare of an environemnt.
+  - r
 dependencies:
   # Python
   - python ==3.7.10
   - pandas ==1.3.1
   - numpy ==1.21.2
+
   # Tools
-  - picard ==2.23.0
-  - r-tidyverse ==1.2.1
-  - snakemake-wrapper-utils >=0.1.3
+  - picard ==2.27.4
+
+  # Need tidyverse because of issue #9
+  # We are now using the version that gets installed when specifying no version,
+  # in the hope that this can be solved more easilty by conda then.
+  - r-tidyverse ==1.3.2
+  - snakemake-wrapper-utils

envs/prep.yaml

@@ -15,4 +15,4 @@ dependencies:
   # Tools
   - bwa ==0.7.17
   - gatk4 ==4.1.4.1
-  - samtools ==1.12
+  - samtools ==1.16.1

envs/qualimap.yaml

@@ -1,5 +1,12 @@
 channels:
-  - bioconda
+  # This env with a single package used to be unsolvable by conda (worked fine with mamba though).
+  # Trying to solve this as described in https://github.com/bioconda/bioconda-utils/issues/330
+  # by using conda-forge as the first channel. Seems to be working now, but only with the specified
+  # qualimap version 2.2.2d below... the others still don't work on conda...
   - conda-forge
+  - bioconda
 dependencies:
-  - qualimap ==2.2.2a
+  - qualimap ==2.2.2d
+  # - qualimap
+  # - qualimap ==2.2.2a
+  # - qualimap ==2.2.2d

envs/rbt.yaml

@@ -3,4 +3,4 @@ channels:
   - conda-forge
 dependencies:
   - rust-bio-tools =0.6.0
-  - bcftools =1.9
+  - bcftools ==1.16

envs/samtools.yaml

@@ -7,5 +7,5 @@ dependencies:
   - pandas ==1.3.1
   - numpy ==1.21.2
   # Tools
-  - samtools ==1.12
+  - samtools ==1.16.1
   - snakemake-wrapper-utils==0.2.0

envs/snpeff.yaml

@@ -8,5 +8,5 @@ dependencies:
   - numpy ==1.21.2
   # Tools
   - snpeff ==4.3.1t
-  - bcftools =1.11
+  - bcftools ==1.16
   - snakemake-wrapper-utils >=0.1.3

envs/tabix.yaml

@@ -2,4 +2,5 @@ channels:
   - bioconda
   - conda-forge
 dependencies:
-  - htslib ==1.10
+  - samtools ==1.16.1
+  # - htslib ==1.16

envs/vcflib.yaml

@@ -4,5 +4,5 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - vcflib =1.0.2
-  - samtools ==1.12
+  - vcflib =1.0.3
+  - samtools ==1.16.1

envs/vep.yaml

@@ -6,8 +6,23 @@ dependencies:
   - python ==3.7.10
   - pandas ==1.3.1
   - numpy ==1.21.2
+
   # Tools
-  # This version should not be changed unless the vep_install command needs an
-  # update or suddenly stops working.
+  # The VEP version should not be changed unless the vep_install command needs an
+  # update or suddenly stops working. It's super fragile...
   - ensembl-vep =104
+  # Unfortunately, for that reason, we also have to keep the bcftools version at this here,
+  # instead of using the more recent version that we have throughout the other env files.
+  # Updating this to bcftools 1.16 conflicts with VEP 104, but updating VEP to a non-conflicting
+  # version then leads to errors as in https://github.com/Ensembl/ensembl-vep/issues/47
+  # For more inspiration on environment configurations that might work, see the official wrapper:
+  # https://github.com/snakemake/snakemake-wrappers/blob/master/bio/vep/annotate/environment.yaml
+  # but that still would need adapting our rules to the new VEP version, updating server paths, etc...
   - bcftools =1.10
+
+  # A potential solution to get the packages to work with the above described updates is to
+  # also add the following two packages. That works to create the env and to get VEP to run,
+  # but then it fails downstream when trying to run the annotate command, with some weird
+  # issues concerning compression. So for now, we don't change the running system.
+  # - perl-dbi
+  # - perl-dbd-mysql

rules/calling-bcftools-combined.smk

@@ -100,6 +100,8 @@ rule called_vcf_index:
         "-p vcf"
     log:
         "logs/tabix/called/{contig}.log"
+    conda:
+        "../envs/tabix.yaml"
     wrapper:
         "0.55.1/bio/tabix"
 

rules/prep.smk

@@ -219,6 +219,8 @@ rule variants_vcf_index:
         "known_variants"
     log:
         os.path.join( variant_logdir, os.path.basename(variants) + ".vcf_index.log" )
+    conda:
+        "../envs/tabix.yaml"
     wrapper:
         "0.55.1/bio/tabix"
 

rules/qc-bam.smk

@@ -32,6 +32,8 @@ rule samtools_stats:
         "benchmarks/samtools-stats/{sample}.bench.log"
     group:
         "qc"
+    conda:
+        "../envs/samtools.yaml"
     wrapper:
         "0.27.1/bio/samtools/stats"
 
@@ -57,6 +59,8 @@ rule samtools_flagstat:
         "benchmarks/samtools-flagstat/{sample}.bench.log"
     group:
         "qc"
+    conda:
+        "../envs/samtools.yaml"
     wrapper:
         "0.64.0/bio/samtools/flagstat"