Merge pull request #91 from genxnetwork/develop

GRAPE v1.7 Release

.github/workflows/pytest.yml

@@ -0,0 +1,27 @@
+name: Run Python Tests
+
+on:
+  pull_request:
+    branches: [ master ]
+    types: [ opened, reopened ]
+
+  workflow_dispatch:
+
+jobs:
+  test:
+    # Self-hosted runner
+    runs-on: self-hosted
+    timeout-minutes: 1000
+    if: |
+      !github.event.pull_request.head.repo.fork &&
+      github.head_ref == 'develop'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.9.13
+      - run: |
+          python -m pip install --upgrade pip
+          pip install -r test/requirements.txt
+      - run: |
+          pytest test/test.py

.github/workflows/verify_files.yml

@@ -3,16 +3,16 @@ name: verify-files
 on:
   schedule:
     - cron: "0 0 * * *"
-    
+
   workflow_dispatch:
 
 jobs:
   verify-files:
     runs-on: ubuntu-latest
-    
+
     steps:
       - uses: actions/checkout@v2
-      
+
       - name: Setup python
         run: "python -m pip install pyyaml"
 

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [v1.7] - 2023-01-12
+
+### Added
+
+- All conda environments are now built in Dockerfile and Snakemake doesn't need to create them for every workflow run from `.yaml` files.
+- Multiple tests were added, covering all GRAPE flows. Test cases are stored at `grape/test-cases`.
+
+### Changed
+
+- Phased affymetrix chip is now stored within the bundle to speed up the simulation flow, because of this `intersect` rule in `pedsim` simulation workflow was moved to `reference` downloading workflow.
+
+### Fixed
+
+- Fixed `ibis` detecting empty IBD segments causing pipeline teardown.
+
 ## [v1.6] - 2022-05-20
 
 ### Added

config.yaml

@@ -36,6 +36,8 @@ reference:
     url: https://ftp-trace.ncbi.nih.gov/hapmap/recombination/2011-01_phaseII_B37/genetic_map_HapMapII_GRCh37.tar.gz
     filesize: 37730100
     md5: 1bc10a34d985e68e1f38ceb137b87929
+  phased_ref:
+    file: 1000genome/phased/chr{chrom}.phased.vcf.gz
   vcfRef:
     file: 1000genome/bcf/1000genome_chr{chrom}.bcf
     url: https://dataset1000genomes.blob.core.windows.net/dataset/release/20130502/supporting/bcf_files/ALL.chr$chrom.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.bcf
@@ -96,9 +98,9 @@ reference:
     md5: 39e6e8620d616362875f2538eae2f279
   bundle:
     file: ref.tar.gz
-    url: https://bioinformatics.file.core.windows.net/bundles/ref.tar.gz
-    filesize: 18582485368
-    md5: 67278f83139f375e22bd56544d523fa3
+    url: https://bioinformatics.file.core.windows.net/bundles/ref_v2.tar.gz
+    filesize: 19411359855
+    md5: 300fa3e768b677958a2b8e270115c6d9
   bundle_min:
     file: ref_min.tar.gz
     url: https://bioinformatics.file.core.windows.net/bundles/ref_min.tar.gz

containers/snakemake/Dockerfile

@@ -5,18 +5,18 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get clean && apt-get update && apt-get install -y ca-certificates libseccomp-dev squashfs-tools \
     && rm -rf /tmp/*
 
-ADD envs/snakemake.yaml envs/snakemake.yaml
+ADD envs envs
 
 ENV PATH /opt/conda/bin:${PATH}
 ENV LANG C.UTF-8
 ENV SHELL /bin/bash
 
-RUN apt-get install -y wget bzip2 gnupg2 git libgomp1 && \
+RUN apt-get update && apt-get install -y wget bzip2 gnupg2 git libgomp1 && \
     wget -nv https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \
     rm Miniconda3-latest-Linux-x86_64.sh && \
     conda install -c conda-forge mamba && \
-    mamba env create -f envs/snakemake.yaml && \
+    for e in envs/*; do mamba env create -f $e ; done && \
     conda clean --all -y
 
 # Intall Minimac3

envs/evaluation.yaml

@@ -6,4 +6,6 @@ dependencies:
     - seaborn==0.10.1
     - matplotlib==3.3.1
     - pandas==1.1.1
-    - pydot==1.4.2
+    - pydot==1.4.2
+    - mmh3==3.0.0
+    - numpy==1.20

envs/interpolation.yaml

@@ -1,3 +1,4 @@
+name: interpolation
 channels:
     - bioconda
     - conda-forge
@@ -7,4 +8,4 @@ dependencies:
     - samtools==1.9
     - openssl==1.0.2u
     - pandas==1.1.1
-    - numpy==1.19.1
+    - numpy==1.19.1

envs/postprocess.yaml

@@ -1,3 +1,4 @@
+name: postprocess
 channels:
     - bioconda
     - conda-forge

envs/snakemake.yaml

@@ -6,7 +6,6 @@ channels:
   - defaults
 dependencies:
   - python>=3.5
-  - snakemake==7.15.2
   - matplotlib==3.3.1
   - pandas==1.1.1
   - numpy==1.19.1
@@ -15,3 +14,8 @@ dependencies:
   - scikit-bio==0.5.6
   - docutils==0.16
   - mmh3==3.0.0
+  - pip:
+    - "--editable=git+https://github.com/Jahysama/snakemake.git#egg=snakemake"
+    #Fork of a snakemake is used here because of not working conda envs inside python scripts
+    #Please check out https://github.com/snakemake/snakemake/pull/1812 for more info
+

envs/vcf_to_ped.yaml

@@ -1,6 +1,7 @@
-name: vcf_to_ped2
+name: vcf_to_ped
 channels:
     - conda-forge
     - defaults
 dependencies:
     - scikit-allel==1.3.2
+    - numpy==1.20

envs/weight-mask.yaml

@@ -2,6 +2,6 @@ name: weight-mask
 channels:
   - conda-forge
 dependencies:
-  - numpy
+  - numpy==1.20
   - scikit-learn
   - matplotlib

launcher.py

@@ -287,7 +287,7 @@ def get_parser_args():
         raise ValueError('If --impute is present, then --phase must also be present')
 
     if args.command != 'reference' and args.use_bundle:
-        raise ValueError('--bundle option only available for reference downloading')
+        raise ValueError('--use-bundle option only available for reference downloading')
 
     if args.num_batches > args.cores:
         raise ValueError('Number of batches is bigger than number cores, please change --num-batches value to be lower or equal --cores')

rules/filter.smk

@@ -7,7 +7,7 @@ rule vcf_stats:
     params:
         samples='vcf/{batch}_merged_lifted.vcf.samples'
     conda:
-        '../envs/bcftools.yaml'
+        'bcftools'
     shell:
         """
             bcftools query --list-samples {input.vcf} > {params.samples}
@@ -29,7 +29,7 @@ rule select_bad_samples:
         alt_hom_samples = config['alt_hom_samples'],
         het_samples = config['het_samples']
     conda:
-        '../envs/evaluation.yaml'
+        'evaluation'
     script:
         '../scripts/select_bad_samples.py'
 
@@ -43,7 +43,7 @@ rule plink_filter:
         bim = temp('plink/{batch}_merged_filter.bim'),
         fam = temp('plink/{batch}_merged_filter.fam')
     conda:
-        '../envs/plink.yaml'
+        'plink'
     params:
         input   = '{batch}_merged',
         out     = '{batch}_merged_filter',
@@ -90,7 +90,7 @@ rule plink_clean_up:
         input = 'plink/{batch}_merged_filter',
         out = 'plink/{batch}_merged_mapped'
     conda:
-        '../envs/plink.yaml'
+        'plink'
     log:
         'logs/plink/{batch}_plink_clean_up.log'
     benchmark:
@@ -128,7 +128,7 @@ rule prepare_vcf:
         input   = 'plink/{batch}_merged_mapped',
         vcf     = 'vcf/{batch}_merged_mapped_sorted.vcf.gz'
     conda:
-         '../envs/bcf_plink.yaml'
+         'bcf_plink'
     log:
         plink='logs/plink/{batch}_prepare_vcf.log',
         vcf='logs/vcf/{batch}_prepare_vcf.log'

rules/imputation.smk

@@ -29,7 +29,7 @@ rule imputation_filter:
     # TODO: because "The option is currently used only for the compression of the output stream"
     # threads: workflow.cores
     conda:
-        '../envs/bcftools.yaml'
+        'bcftools'
     log:
         'logs/impute/{batch}_imputation_filter-{chrom}.log'
     benchmark:
@@ -63,7 +63,7 @@ rule merge_imputation_filter:
         mode=config['mode'],
         merged_imputed = 'background/' + card + '_merged_imputed.vcf.gz'
     conda:
-        '../envs/bcftools.yaml'
+        'bcftools'
     log:
         'logs/vcf/' + card + '_merge_imputation_filter.log'
     benchmark:
@@ -103,7 +103,7 @@ rule convert_imputed_to_plink:
     params:
         out = 'plink/{batch}_merged_imputed'
     conda:
-        '../envs/plink.yaml'
+        'plink'
     log:
         'logs/plink/{batch}_convert_imputed_to_plink.log'
     benchmark:
@@ -126,7 +126,7 @@ rule merge_convert_imputed_to_plink:
         background  = 'background/{batch}_merged_imputed',
         out         = 'plink/client/{batch}_merged_imputed'
     conda:
-        '../envs/plink.yaml'
+        'plink'
     log:
         'logs/plink/{batch}_convert_imputed_to_plink.log'
     benchmark:

rules/phasing.smk

@@ -31,7 +31,7 @@ rule merge_phased:
         list='vcf/{batch}_phased.merge.list',
         mode=config['mode']
     conda:
-        '../envs/bcftools.yaml'
+        'bcftools'
     log:
         'logs/vcf/{batch}_merged_phased.log'
     benchmark: