Merge pull request #1340 from nf-core/add-azure-megatests

Add Azure megatests

.github/workflows/cloudtest.yml

@@ -0,0 +1,103 @@
+name: nf-core cloud test
+
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+    inputs:
+      test:
+        description: "-profile test (smaller)"
+        type: boolean
+        default: true
+      somatic:
+        description: "Somatic full test"
+        type: boolean
+        default: false
+      germline:
+        description: "Germline full test"
+        type: boolean
+        default: false
+      aws:
+        description: "AWS Batch"
+        type: boolean
+        default: true
+      azure:
+        description: "Azure Batch"
+        type: boolean
+        default: true
+
+jobs:
+  trigger-profile-test:
+    name: Run AWS tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - profile: test_aws
+            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.aws ) ) }}
+          - profile: test_azure
+            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.azure ) ) }}
+          - profile: test_full_aws
+            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.aws ) ) }}
+          - profile: test_full_azure
+            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.azure ) ) }}
+          - profile: test_full_germline_aws
+            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.aws ) ) }}
+          - profile: test_full_germline_azure
+            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.azure ) ) }}
+
+    steps:
+      # Launch workflow on AWS Batch
+      - name: AWS Launch
+        uses: seqeralabs/action-tower-launch@v2
+        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.aws ) }}
+        with:
+          run_name: sarek_${{ matrix.profile }}
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
+          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/${{ matrix.profile }}
+          parameters: |
+            {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
+            }
+          profiles: ${{ matrix.profile }}
+
+      - uses: actions/upload-artifact@v3
+        name: Save AWS Logs
+        if: success() || failure()
+        with:
+          name: tower-aws-${{ matrix.profile }}-log
+          path: |
+            tower_action_*.log
+            tower_action_*.json
+
+      # Launch workflow using Tower CLI tool action
+      - name: Azure Launch
+        uses: seqeralabs/action-tower-launch@v2
+        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.azure ) }}
+        with:
+          run_name: sarek_${{ matrix.profile }}
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }}
+          revision: ${{ github.sha }}
+          workdir: ${{ secrets.TOWER_BUCKET_AZURE}}/sarek/work-${{ github.sha }}/${{ matrix.profile }}
+          parameters: |
+            {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
+              "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
+            }
+          profiles: ${{ matrix.profile }}
+
+      - uses: actions/upload-artifact@v3
+        name: Save Azure Logs
+        if: success() || failure()
+        with:
+          name: tower-azure-${{ matrix.profile }}-log
+          path: |
+            tower_action_*.log
+            tower_action_*.json

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - [#1333](https://github.com/nf-core/sarek/pull/1333) - Back to dev
 - [#1335](https://github.com/nf-core/sarek/pull/1335) - Add index computation of `bcftools_annotations`, if not provided
+- [#1340](https://github.com/nf-core/sarek/pull/1340) - Adds Azure test profiles and megatests.
 
 ### Changed
 

nextflow.config

@@ -249,10 +249,35 @@ profiles {
     }
     // Basic test profile for CI
     test                       { includeConfig 'conf/test.config' }
+    test_aws                   { 
+        includeConfig 'conf/test.config'
+        params.sentieon_dnascope_model = "s3://ngi-igenomes/igenomes/Homo_sapiens/GATK/GRCh38/Annotation/Sentieon/SentieonDNAscopeModel1.1.model"
+    }
+    test_azure                 { 
+        includeConfig 'conf/test.config' 
+        params.sentieon_dnascope_model = "az://igenomes/Homo_sapiens/GATK/GRCh38/Annotation/Sentieon/SentieonDNAscopeModel1.1.model"
+    }
     test_cache                 { includeConfig 'conf/test/cache.config' }
     // Extra test profiles for full tests on AWS
     test_full                  { includeConfig 'conf/test_full.config' }
+    test_full_aws              { 
+        includeConfig 'conf/test_full.config' 
+    }
+    test_full_azure            { 
+        includeConfig 'conf/test_full.config'
+        params.input         = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/HCC1395_WXS_somatic_full_test_azure.csv'
+        params.intervals     = 'az://test-data/sarek/S07604624_Padded_Agilent_SureSelectXT_allexons_V6_UTR.bed'
+        params.igenomes_base = "az://igenomes"
+    }
     test_full_germline         { includeConfig 'conf/test_full_germline.config' }
+    test_full_germline_aws     { 
+        includeConfig 'conf/test_full_germline.config' 
+    }
+    test_full_germline_azure   { 
+        includeConfig 'conf/test_full_germline.config'
+        params.input         = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/NA12878_WGS_30x_full_test_azure.csv'
+        params.igenomes_base = "az://igenomes"
+    }
     // Extra test profiles for more complete CI
     alignment_to_fastq         { includeConfig 'conf/test/alignment_to_fastq.config' }
     annotation                 { includeConfig 'conf/test/annotation.config' }