Merge fd88b30 into 5b46cef

.github/workflows/build_and_publish_latest_docker.yml

@@ -0,0 +1,25 @@
+---
+
+name: Publish bleeding edge to Docker Hub
+
+"on":
+  push:
+    branches:
+      - develop
+
+jobs:
+  docker-image-CI:
+    name: Docker Image CI
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out git repository
+        uses: actions/checkout@v2
+
+      - name: Build and publish to Docker Hub
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: clinicalgenomics/mip
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tags: "latest"

.github/workflows/build_and_publish_prod_docker.yml

@@ -0,0 +1,25 @@
+---
+
+name: Publish production release to Docker Hub
+
+"on":
+  release:
+    types:
+      - created
+
+jobs:
+  docker-image-CI:
+    name: Docker Image CI
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out git repository
+        uses: actions/checkout@v2
+
+      - name: Build and publish to Docker Hub
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: clinicalgenomics/mip
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tags: "${{ github.event.release.tag_name }}"

.github/workflows/conda_prod_install.yml

@@ -29,8 +29,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda activate mip_ci
-          perl mip install --bash_set_errexit --envn mip_ci --select_program mip_scripts python
-          bash mip.sh
+          perl mip install --envn mip_ci --select_program mip_scripts
 
       - name: Install bcftools in conda test env
         shell: bash -l {0}
@@ -43,12 +42,11 @@ jobs:
           conda activate mip_ci
           prove -lrs -f -j 2 t
 
-      - name: Run MIP download tests
+      - name: Run MIP download test
         shell: bash -l {0}
         run: |
           conda activate mip_ci
-          perl t/mip_download_rd_dna.test
-          perl t/mip_download_rd_rna.test
+          perl t/mip_download.test
 
       - name: Run MIP analyse tests
         shell: bash -l {0}
@@ -66,4 +64,3 @@ jobs:
           conda activate mip_ci
           perl t/mip_qccollect.test
           perl t/mip_vcfparser.test
-          perl t/mip_vercollect.test

.github/workflows/coverage.yml

@@ -23,33 +23,31 @@ jobs:
           cpanm --quiet --notest --installdeps .
           cpanm --quiet --notest Devel::Cover::Report::Coveralls
 
+      - name: Install bcftools in conda test env
+        shell: bash -l {0}
+        run: |
+          conda create -n mip_ci --channel bioconda --channel conda-forge bcftools=1.9=ha228f0b_4
+
       - name: Run MIP install test
         shell: bash -l {0}
         run: PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl t/mip_install.test
 
       - name: Install MIP rare disease
         shell: bash -l {0}
         run: |
-          PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl mip install --bash_set_errexit --envn mip_ci --select_program mip_scripts python
-          bash mip.sh
-
-      - name: Install bcftools in conda test env
-        shell: bash -l {0}
-        run: |
-          conda install -n mip_ci --channel bioconda --channel conda-forge bcftools=1.9=ha228f0b_4
+          PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl mip install --envn mip_ci --select_program mip_scripts
 
       - name: Run MIP unit tests
         shell: bash -l {0}
         run: |
           conda activate mip_ci
           PERL5OPT=-MDevel::Cover=-ignore,"^t/",+ignore,"prove",-coverage,statement,branch,condition,path,subroutine prove -lrs -f -j 2 t
 
-      - name: Run MIP download tests
+      - name: Run MIP download test
         shell: bash -l {0}
         run: |
           conda activate mip_ci
-          PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl t/mip_download_rd_dna.test
-          PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl t/mip_download_rd_rna.test
+          PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl t/mip_download.test
 
       - name: Run MIP analyse tests
         shell: bash -l {0}
@@ -67,7 +65,6 @@ jobs:
           conda activate mip_ci
           PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl t/mip_qccollect.test
           PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl t/mip_vcfparser.test
-          PERL5OPT=-MDevel::Cover=-ignore,"^t/",-coverage,statement,branch,condition,path,subroutine perl t/mip_vercollect.test
 
       - name: Calculate coveralls
         run: |

.github/workflows/testing.yml

@@ -22,33 +22,31 @@ jobs:
           perl -V
           cpanm --quiet --notest --installdeps .
 
+      - name: Install bcftools in conda test env
+        shell: bash -l {0}
+        run: |
+          conda create -n mip_ci --channel bioconda --channel conda-forge bcftools=1.9=ha228f0b_4
+
       - name: Run MIP install test
         shell: bash -l {0}
         run: perl t/mip_install.test
 
       - name: Install MIP rare disease
         shell: bash -l {0}
         run: |
-          perl mip install --bash_set_errexit --envn mip_ci --select_program mip_scripts python
-          bash mip.sh
-
-      - name: Install bcftools in conda test env
-        shell: bash -l {0}
-        run: |
-          conda install -n mip_ci --channel bioconda --channel conda-forge bcftools=1.9=ha228f0b_4
+          perl mip install --envn mip_ci --select_program mip_scripts
 
       - name: Run MIP unit tests
         shell: bash -l {0}
         run: |
           conda activate mip_ci
           prove -lrs -f -j 2 t
 
-      - name: Run MIP download tests
+      - name: Run MIP download test
         shell: bash -l {0}
         run: |
           conda activate mip_ci
-          perl t/mip_download_rd_dna.test
-          perl t/mip_download_rd_rna.test
+          perl t/mip_download.test
 
       - name: Run MIP analyse tests
         shell: bash -l {0}
@@ -66,4 +64,3 @@ jobs:
           conda activate mip_ci
           perl t/mip_qccollect.test
           perl t/mip_vcfparser.test
-          perl t/mip_vercollect.test

.gitignore

@@ -7,3 +7,10 @@ mip_log/
 **/.idea
 _Deparsed_XSubs.pm
 .gitignore
+**/.vscode
+
+## Carton
+local/
+
+## Carmel
+.carmel/

.perlcriticrc_mip → .perlcriticrc

@@ -1,4 +1,5 @@
 ## Modifications to perl critic policies for MIP development
+severity = 1
 
 ## Increase the number of lines between open and close filehandle, default 9
 [InputOutput::RequireBriefOpen]
@@ -8,6 +9,8 @@ lines = 100
 functions = :builtins
 exclude_functions = print say
 
+[-Perl::Critic::Policy::Modules::RequireVersionVar]
+
 ## Perltidy has already been run with a diffrent slightlty modified rules
 [-CodeLayout::RequireTidyCode]
 

.perltidyrc

@@ -0,0 +1,6 @@
+## Modifications to perl tidy policies for MIP development
+--maximum-line-length=100
+--iterations=2
+--backup-and-modify-in-place
+--backup-file-extension='/'
+--standard-error-output

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+---
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    - id: trailing-whitespace
+      args: [--markdown-linebreak-ext=md]
+    - id: check-merge-conflict
+  - repo: https://github.com/perltidy/perltidy
+    rev: '20201207'
+    hooks:
+    - id: perltidy
+      types: [file]
+      files: \.(t|pm|pl|test)$
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.25.0
+    hooks:
+    - id: yamllint
+  - repo: local
+    hooks:
+    - id: perlcritic
+      name: perlcritc
+      language: system
+      entry: perlcritic
+      types: [file]
+      files: \.(t|pm|pl|test)$

CHANGELOG.md

@@ -1,7 +1,61 @@
 # Change Log
+
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [10.0.0]
+
+- Remove unused recipe split_fastq_file
+- Align with the same bwa mem options as used by Broad
+  - Fixed number of bases in each batch
+  - Use bwa mem instead of run-bwamem for alignment to grch38
+- Use Chanjo repos Docker file instead of MIPs
+- Removed support to run bcftools_mpileup as a variant caller
+- Added perl and MIP docker file and use it in recipes
+- Removed PATH check for proxy bins
+- Switched to using container manager instead of proxy bins in recipes
+- Removed support to run variant_integrity
+- Added deepvariant as variant caller and glnexus to merge samples to case vcf
+
+### Tools
+
+arriba: v1.2.0 -> v2.1.0
+bedtools: 2.29.0 -> 2.30.0
+bcftools: 1.10.2-hd2cd319_0 -> 1.12=h45bccc9_1
+bwa-mem2 2.0 -> 2.2
+chanjo: 4.2.0 -> 4.6
+chromograph: 1.0.1 -> 1.1
+cyrius: v1.0 -> v1.1
+deepvariant: 1.1.0
+delly: 0.8.1 -> 0.8.7
+gatk: v4.1.8.1 -> v4.2.0.0
+glnexus: v1.3.1
+htslib: 1.10.2-h78d89cc_0 -> 1.12=h9093b5e_1
+megafusion: 66a3a80
+multiqc: 1.9 -> 1.10.1
+pdfmerger: v1.0
+picardtools: 2.23.4 -> 2.25.0
+preseq: 2.0.3 -> 3.1.2
+rseqc: 3.0.1 -> 4.0.0
+rtg-tools: 3.10.1 -> 3.12
+salmon: 0.12.0 -> 1.4.0
+samtools: 1.10-h9402c20_2 -> 1.12=h9aed4be_1
+smncopynumbercaller: 4b2c1ad -> v1.1.1
+star: 2.7.4a -> 2.7.8a
+stranger: 0.7 -> 0.7.1
+svdb: 2.2.0 -> 2.4.0--py37h77a2a36_4
+telomerecat: 3.4.0 -> 4.0.0
+tiddit: 2.8.1 -> 2.12.1
+upd: 0.1 -> 0.1.1
+
+### References
+
+- clinvar_20200905 -> clinvar_20210415
+- VEP cache: 100 -> 103.1
+- grch37_gencode_v19_ctat_lib_plug-n-play_-apr032020-.tar.gz -> grch37_gencode_v19_CTAT_lib_Mar012021.plug-n-play.tar.gz
+- grch38_gencode_v31_ctat_lib_plug-n-play_-apr062020-.tar.gz -> grch38_gencode_v37_CTAT_lib_Mar012021.plug-n-play.tar.gz
+- gencode annotation: v34 -> v37
+
 ## [9.1.3]
 - Fix memory allocation for mip-rna markduplicates.
 - Update to repeat expansion calling
@@ -41,7 +95,7 @@ chromograph 0.3.3 -> 1.0.0
 - Update to chromograph in order to fix the renderering coverage images
 
 **Tools**
-chromograph 0.3.1 -> 0.3.3
+chromograph: 0.3.1 -> 0.3.3
 
 ## [9.0.4]
 - Increased memory allocation for samtools_subsample_mt

CONTRIBUTING.md

@@ -1,16 +1,46 @@
 ## Versioning
-MIP adheres to [semantic versioning](https://semver.org/).
+MIP adheres to [semantic versioning].
 
 ## Branching model
-MIP follows a gitflow [branching model](http://www.clinicalgenomics.se/development/dev/gitflow/).
+MIP follows a gitflow [branching model].
+
+## Dependencies
+We use [Carton] to lock MIP's perl dependencies in the `cpanfile.snapshot`file.
+
+### Initializing the environment
+Carton will use the local directory to install modules into. You're recommended to exclude these directories from git by adding the `local/` to `.gitignore`.
+After installing Carton with `cpanm install Carton`, you can create/update the snapshot file by:
+
+```
+carton install
+```
+
+Once you've done installing all the dependencies, you can push your application directory to a remote machine (excluding local and .carton) and run the following command:
+
+```
+carton install --deployment
+```
+
+This will look at the cpanfile.snapshot and install the exact same versions of the dependencies into local, and now your application is ready to run.
+
+The --deployment flag makes sure that carton will only install modules and versions available in your snapshot, and won't fallback to query for CPAN Meta DB for missing modules.
+
+Another flavor of this is [Carmel]. Unlike traditional CPAN module installer, Carmel keeps the build of your dependencies in a central repository, then select the library paths to include upon runtime.
 
 ## Code style
-MIP uses Perl::Tidy, Perl::Critic and Yamllint in order to maintain readability and a consistent code style across the repo. These tools can be installed by running mip_install_perl.sh with the `-d` option when installing MIP's perl distribution and cpan modules.  
-
-Prior to opening a pull request each commit should be tested with the supplied bash script `mip-check`. Running `bash mip-check` will test the code newly edited code using perltidy, perlcritic and yamllint.
+MIP uses Perl::Tidy, Perl::Critic and Yamllint in order to maintain readability and a consistent code style across the repo. The repo uses [pre-commit] to ensure that commits have been formatted correctly. You can set this up yourself with  setup with `pre-commit install`.
+Alternatively by using the `-d` flag with the mip_install_perl.sh script the extra dependencies will be installed for use with your IDE. The flag also ensures that pre-commit is installed in the develop environment and setups pre-commit to autoamtically run on commits.
+
+Each commit will be tested with these tools using pre-commit.
+
+ - Perltidy will automatically check and reformat your perl code.
+ - Perlcritic will lint the code and ensure that it follows perl best practice.
+ - Yamllint check any yaml files that have been altered.
 
- - Perltidy will automatically check and reformat your perl code. 
- - Perlcritic will lint the code and ensure that it follows perl best practice. 
- - Yamllint check any yaml files that have been altered. 
+Try to keep the number of warnings to a minimum. This ensures that your new code adheres to the style that has been agreed upon for the MIP repo. The pre-commit hooks can be disabled by using the flag `-n`, i.e. `git commit -n`
 
-Try to keep the number of warnings to a minimum. This ensures that your new code adheres to the style that has been agreed upon for the MIP repo.
+[branching model]: http://www.clinicalgenomics.se/development/dev/gitflow/
+[Carmel]: https://metacpan.org/pod/Carmel
+[Carton]: https://metacpan.org/pod/Carton
+[semantic versioning]: https://semver.org/
+[pre-commit]: https://pre-commit.com