refactor: adjust final report; add full region scanning in CSM (#34) #240
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ############################################################################### | |
| # | |
| # GitHub Actions CI configuration for integration tests | |
| # | |
| # AUTHOR: Maciej_Bak | |
| # AFFILIATION: University_of_Basel | |
| # AFFILIATION: Swiss_Institute_of_Bioinformatics | |
| # CONTACT: wsciekly.maciek@gmail.com | |
| # CREATED: 14-12-2021 | |
| # LICENSE: Apache_2.0 | |
| # | |
| ############################################################################### | |
| --- | |
| name: integration | |
| on: | |
| push: | |
| branches: | |
| - '*' | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| PQA-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/PREPROCESSING/execution/run.sh \ | |
| -c .test/integration/module_PQA/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/PREPROCESSING/Snakefile \ | |
| --configfile=.test/integration/module_PQA/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/PREPROCESSING/Snakefile \ | |
| --configfile=.test/integration/module_PQA/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| ASE-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/EXTRACT_AS_EXONS/execution/run.sh \ | |
| -c .test/integration/module_ASE/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/EXTRACT_AS_EXONS/Snakefile \ | |
| --configfile=.test/integration/module_ASE/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/EXTRACT_AS_EXONS/Snakefile \ | |
| --configfile=.test/integration/module_ASE/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| TPA-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/PREPARE_TANDEM_PAS/execution/run.sh \ | |
| -c .test/integration/module_TPA/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/PREPARE_TANDEM_PAS/Snakefile \ | |
| --configfile=.test/integration/module_TPA/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/PREPARE_TANDEM_PAS/Snakefile \ | |
| --configfile=.test/integration/module_TPA/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| QEI-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/QUANTIFICATION/execution/run.sh \ | |
| -c .test/integration/module_QEI/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/QUANTIFICATION/Snakefile \ | |
| --configfile=.test/integration/module_QEI/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/QUANTIFICATION/Snakefile \ | |
| --configfile=.test/integration/module_QEI/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| PAQ-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/PAQR/execution/run.sh \ | |
| -c .test/integration/module_PAQ/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/PAQR/Snakefile \ | |
| --configfile=.test/integration/module_PAQ/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/PAQR/Snakefile \ | |
| --configfile=.test/integration/module_PAQ/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| CSM-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| # Test 1: kmers | |
| - name: "Test 1: Create snakemake DAG" | |
| run: | | |
| bash modules/CREATE_SITECOUNT_MATRICES/execution/run.sh \ | |
| -c .test/integration/module_CSM/config1_kmers.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| # | |
| - name: "Test 1: Initiate snakemake dry run" | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \ | |
| --configfile=.test/integration/module_CSM/config1_kmers.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| # | |
| - name: "Test 1: Build conda envs for all the rules" | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \ | |
| --configfile=.test/integration/module_CSM/config1_kmers.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| # Test 2: pwms | |
| - name: "Test 2: Create snakemake DAG" | |
| run: | | |
| bash modules/CREATE_SITECOUNT_MATRICES/execution/run.sh \ | |
| -c .test/integration/module_CSM/config2_pwms.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| # | |
| - name: "Test 2: Initiate snakemake dry run" | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \ | |
| --configfile=.test/integration/module_CSM/config2_pwms.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| # | |
| - name: "Test 2: Build conda envs for all the rules" | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/CREATE_SITECOUNT_MATRICES/Snakefile \ | |
| --configfile=.test/integration/module_CSM/config2_pwms.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| MAE-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/MAEI/execution/run.sh \ | |
| -c .test/integration/module_MAE/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/MAEI/Snakefile \ | |
| --configfile=.test/integration/module_MAE/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/MAEI/Snakefile \ | |
| --configfile=.test/integration/module_MAE/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| KPC-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/KAPAC/execution/run.sh \ | |
| -c .test/integration/module_KPC/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/KAPAC/Snakefile \ | |
| --configfile=.test/integration/module_KPC/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/KAPAC/Snakefile \ | |
| --configfile=.test/integration/module_KPC/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| RES-module: | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Create snakemake DAG | |
| run: | | |
| bash modules/REPORT_RESULTS/execution/run.sh \ | |
| -c .test/integration/module_RES/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/REPORT_RESULTS/Snakefile \ | |
| --configfile=.test/integration/module_RES/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| - name: Build conda envs for all the rules | |
| run: | | |
| snakemake \ | |
| --snakefile=modules/REPORT_RESULTS/Snakefile \ | |
| --configfile=.test/integration/module_RES/config.yml \ | |
| --use-conda \ | |
| --conda-create-envs-only \ | |
| --cores 8 | |
| MAPP-test1: | |
| needs: | |
| - PQA-module | |
| - ASE-module | |
| - TPA-module | |
| - QEI-module | |
| - PAQ-module | |
| - CSM-module | |
| - MAE-module | |
| - KPC-module | |
| - RES-module | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Download genomic resources | |
| run: | | |
| bash scripts/download-ENSEMBL-resources.sh \ | |
| --species hsa \ | |
| --output-directory resources_ENSEMBL_hsa | |
| - name: Download the atlas of PolyA sites | |
| run: | | |
| bash scripts/download-polyA-atlas.sh \ | |
| --species hsa \ | |
| --output-directory ATLAS2_hsa | |
| - name: Download and parse ATtRACT motifs | |
| run: | | |
| bash scripts/download-and-parse-ATtRACT-motifs.sh \ | |
| --output-directory ATtRACT_hsa | |
| - name: Create the main configfile | |
| run: | | |
| python scripts/create-main-config-file.py \ | |
| --config-template .test/integration/MAPP/config_template_1.yml \ | |
| --pipeline-configfile configs/config.yml | |
| - name: Create snakemake rulegraph | |
| run: | | |
| bash execution/run.sh \ | |
| -c configs/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g rulegraph rulegraph.svg | |
| - name: Create snakemake DAG | |
| run: | | |
| bash execution/run.sh \ | |
| -c configs/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=Snakefile \ | |
| --configfile=configs/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| MAPP-test2: | |
| needs: | |
| - PQA-module | |
| - ASE-module | |
| - TPA-module | |
| - QEI-module | |
| - PAQ-module | |
| - CSM-module | |
| - MAE-module | |
| - KPC-module | |
| - RES-module | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Checkout mapp repository | |
| uses: actions/checkout@v2 | |
| - name: Setup miniconda & mapp main environment | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: false | |
| miniforge-variant: Mambaforge | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: mapp | |
| environment-file: env/environment.yml | |
| auto-activate-base: false | |
| - name: Display all miniconda & env info | |
| run: | | |
| conda info -a | |
| conda list | |
| - name: Download genomic resources | |
| run: | | |
| bash scripts/download-ENSEMBL-resources.sh \ | |
| --species hsa \ | |
| --output-directory resources_ENSEMBL_hsa | |
| - name: Download the atlas of PolyA sites | |
| run: | | |
| bash scripts/download-polyA-atlas.sh \ | |
| --species hsa \ | |
| --output-directory ATLAS2_hsa | |
| - name: Download and parse ATtRACT motifs | |
| run: | | |
| bash scripts/download-and-parse-ATtRACT-motifs.sh \ | |
| --output-directory ATtRACT_hsa | |
| # for this test it is required that we set the | |
| # modification date of resources files as older than | |
| # the pipeline files (expected situation): | |
| - name: Adjust modification date for resources | |
| run: | | |
| touch -a -m -t 200001010000.00 resources_ENSEMBL_hsa/Homo_sapiens.GRCh38.102.gtf | |
| touch -a -m -t 200001010000.00 resources_ENSEMBL_hsa/Homo_sapiens.GRCh38.dna.primary_assembly.fa | |
| touch -a -m -t 200001010000.00 ATLAS2_hsa/reformatted_filtered_atlas.bed | |
| # according to the notes in the pipeline configuration: | |
| # comment-out ASE, TPA and CSM modules inclusion from the main Snakefile: | |
| - name: Comment-out ASE, TPA, CSM modules | |
| run: | | |
| sed '52 {s/^/#/}' -i Snakefile | |
| sed '53 {s/^/#/}' -i Snakefile | |
| sed '56 {s/^/#/}' -i Snakefile | |
| - name: Create the main configfile | |
| run: | | |
| python scripts/create-main-config-file.py \ | |
| --config-template .test/integration/MAPP/config_template_2.yml \ | |
| --pipeline-configfile configs/config.yml | |
| - name: Create snakemake rulegraph | |
| run: | | |
| bash execution/run.sh \ | |
| -c configs/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g rulegraph rulegraph.svg | |
| - name: Create snakemake DAG | |
| run: | | |
| bash execution/run.sh \ | |
| -c configs/config.yml \ | |
| -e local \ | |
| -t conda \ | |
| -g dag dag.svg | |
| - name: Initiate snakemake dry run | |
| run: | | |
| snakemake \ | |
| --snakefile=Snakefile \ | |
| --configfile=configs/config.yml \ | |
| --printshellcmds \ | |
| --dryrun \ | |
| --verbose | |
| ... |