diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8fdd2bd7e1..5043b37acc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -37,5 +37,5 @@ body: * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ - * Version of nf-core/tools _(eg. 1.1, 1.5, 1.8.2)_ - * Python version _(eg. 3.10, 3.11)_ + * Version of nf-core/tools _(eg. 1.10, 1.12.1, 1.13)_ + * Python version _(eg. 3.11, 3.12)_ diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 51bf2ae5f7..0bc5e432e7 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -17,7 +17,7 @@ runs: # Set up Nextflow - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: ${{ matrix.NXF_VER }} @@ -77,7 +77,7 @@ runs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py index 4835b0d0f0..471665e4b1 100644 --- a/.github/workflows/changelog.py +++ b/.github/workflows/changelog.py @@ -79,7 +79,7 @@ def _determine_change_type(pr_title) -> tuple[str, str]: section, section_header = _determine_change_type(pr_title) # Remove section indicator from the PR title. -pr_title = re.sub(rf"{section}[:\s]*", "", pr_title, flags=re.IGNORECASE) +pr_title = re.sub(rf"{section}:[\s]*", "", pr_title, flags=re.IGNORECASE) # Prepare the change log entry. pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))" @@ -91,7 +91,6 @@ def _determine_change_type(pr_title) -> tuple[str, str]: new_lines = [ f"- {pr_title} {pr_link}\n", ] - print(f"Adding new lines into section '{section}':\n" + "".join(new_lines)) # Finally, updating the changelog. @@ -144,10 +143,11 @@ def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> st # Parse version from the line `## v2.12dev` or # `## [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]` ... - if not (m := re.match(r".*(v\d+\.\d+(dev)?).*", line)): + if not (m := re.match(r".*(v\d+\.\d+.\d*(dev)?).*", line)): print(f"Cannot parse version from line {line.strip()}.", file=sys.stderr) sys.exit(1) version = m.group(1) + print(f"Found version: {version}") if not inside_version_dev: if not version.endswith("dev"): @@ -209,6 +209,7 @@ def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> st # If the line already contains a link to the PR, don't add it again. line = _skip_existing_entry_for_this_pr(line, same_section=True) section_lines.append(line) + else: updated_lines.append(line) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 63bf8fa5ed..cebcc854bc 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -19,7 +19,7 @@ jobs: ) steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} @@ -36,9 +36,9 @@ jobs: fi gh pr checkout $PR_NUMBER - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install packages run: | @@ -64,10 +64,10 @@ jobs: git diff --exit-code ${GITHUB_WORKSPACE}/CHANGELOG.md || echo "changed=YES" >> $GITHUB_ENV echo "File changed: ${{ env.changed }}" - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index e69229c051..03b9aa2411 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -7,6 +7,9 @@ on: - "docs/**" - "CHANGELOG.md" pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: @@ -45,14 +48,14 @@ jobs: export NXF_WORK=$(pwd) # Get the repo code - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository # Set up nf-core/tools - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: pip - name: Install python dependencies diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 188cccfd75..0de7287a57 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -6,6 +6,9 @@ on: paths: - nf_core/pipeline-template/** pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: @@ -58,13 +61,13 @@ jobs: cd create-lint-wf-template export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | @@ -72,7 +75,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: latest-everything @@ -155,7 +158,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.TEMPLATE }} path: create-test-lint-wf/artifact_files.tar diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index cd36d4e62d..87cdf2e7bb 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -7,6 +7,9 @@ on: - "docs/**" - "CHANGELOG.md" pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: @@ -45,13 +48,13 @@ jobs: cd create-test-wf export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | @@ -59,7 +62,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: ${{ matrix.NXF_VER }} @@ -72,7 +75,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-test-wf/log.txt diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index e53d2f2f5a..1202891e4d 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 95a03c70fe..4334871c4c 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -32,9 +32,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 8ed52a0582..3bddd42d49 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,12 +18,12 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index af11a79990..c613e13a2d 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 0b3f381d86..5a076f6d3b 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index eec676a746..70b9cfd0a8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -12,7 +12,9 @@ on: paths-ignore: - "docs/**" - "CHANGELOG.md" + # ignore github workflows except for the current one - ".github/**" + - "!.github/workflows/pytest.yml" release: types: [published] workflow_dispatch: @@ -38,7 +40,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] runner: ["ubuntu-latest"] include: - python-version: "3.8" @@ -59,7 +61,7 @@ jobs: name: Get test file matrix runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: List tests @@ -85,11 +87,11 @@ jobs: cd pytest export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: ${{ needs.setup.outputs.python-version }} cache: "pip" @@ -119,32 +121,10 @@ jobs: run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - - - name: Look if nf-test is already installed and write to env variable - id: check-nftest - run: | - if [ -f /usr/local/bin/nf-test ]; then - echo "nftest_installed=true" >> $GITHUB_ENV - else - echo "nftest_installed=false" >> $GITHUB_ENV - fi - - - name: Cache nf-test installation - if: env.nftest_installed != 'true' - id: cache-software - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest-${{ env.date }} + uses: nf-core/setup-nextflow@v2 - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' && env.nftest_installed != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ + uses: nf-core/setup-nf-test@v1 - name: move coveragerc file up run: | @@ -163,7 +143,7 @@ jobs: fi - name: Upload coverage - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: coverage_${{ matrix.test }} path: .coverage @@ -178,13 +158,13 @@ jobs: mkdir -p pytest cd pytest - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install dependencies @@ -197,14 +177,14 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85 # v4 + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* coverage report coverage xml - - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4 + - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be # v4 with: files: coverage.xml env: diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 8748f3d7b3..cd12b139d3 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,31 +6,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: 3.x cache: pip cache-dependency-path: setup.py - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - - - name: Cache nf-test installation - id: cache-software - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest-${{ env.date }} + uses: nf-core/setup-nextflow@v2 - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ + uses: nf-core/setup-nf-test@v1 - name: Install nf-core/tools run: pip install git+https://github.com/nf-core/tools.git@dev diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index c998552ddd..23b05de88a 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -15,6 +15,10 @@ on: - "ubuntu-latest" - "self-hosted" default: "self-hosted" + force_pr: + description: "Force a PR to be created" + type: boolean + default: false # Cancel if a newer run is started concurrency: @@ -44,10 +48,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/tools - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} @@ -56,10 +60,10 @@ jobs: path: nf-core/${{ matrix.pipeline }} fetch-depth: "0" - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | @@ -67,7 +71,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "latest-everything" @@ -86,7 +90,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index f357bed03b..5ba513735e 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -11,10 +11,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.x" @@ -23,11 +23,11 @@ jobs: - name: Update modules run: nf-core modules update --all - working-directory: nf-core/pipeline-template + working-directory: nf_core/pipeline-template - name: Update subworkflows run: nf-core subworkflows update --all - working-directory: nf-core/pipeline-template + working-directory: nf_core/pipeline-template # Commit the changes - name: Commit changes diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03fbb7bedf..887cbe027c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.2 + rev: v0.4.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix @@ -9,6 +9,8 @@ repos: rev: "v3.1.0" hooks: - id: prettier + additional_dependencies: + - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "2.7.3" @@ -17,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.8.0" + rev: "v1.10.0" hooks: - id: mypy additional_dependencies: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3189edaa..a7eea54dd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,70 @@ # nf-core/tools: Changelog +## [v2.14.0 - Tantalum Toad](https://github.com/nf-core/tools/releases/tag/2.14) - [2024-05-07] + +### Template + +- Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) +- Update templates to use nf-core/setup-nextflow v2 ([#2818](https://github.com/nf-core/tools/pull/2818)) +- Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) +- Add fallback to `download_pipeline.yml` in case the pipeline does not support stub runs ([#2846](https://github.com/nf-core/tools/pull/2846)) +- Set topic variable correctly in the mastodon announcement ([#2848](https://github.com/nf-core/tools/pull/2848)) +- Add a cleanup action to `download_pipeline.yml` to fix failures caused by inadequate storage space on the runner ([#2849](https://github.com/nf-core/tools/pull/2849)) +- Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) +- Remove `pyproject.toml` from template root +- Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) +- Add a new hidden `--pipelines_testdata_base_path` parameter to more easily switch locations of test data in test configs (#2931)[https://github.com/nf-core/tools/pull/2931] +- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Template config: `conda.channels`, not `channels` ([#2950](https://github.com/nf-core/tools/pull/2950)) +- Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) +- Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) +- Update .editorconfig ([#2953](https://github.com/nf-core/tools/pull/2953)) + +### Linting + +- Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) +- Include test for presence of versions in snapshot ([#2888](https://github.com/nf-core/tools/pull/2888)) +- Components: set correct sha before running component lint tests ([#2952](https://github.com/nf-core/tools/pull/2952)) +- Less strict logo comparison ([#2956](https://github.com/nf-core/tools/pull/2956)) +- Handle request errors more gracefully for actions validation ([#2959](https://github.com/nf-core/tools/pull/2959)) + +### Download + +- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. ([#2853](https://github.com/nf-core/tools/pull/2853)) +- Better error message when GITHUB_TOKEN exists but is wrong/outdated +- New `--tag` argument to add custom tags during a pipeline download ([#2938](https://github.com/nf-core/tools/pull/2938)) + +### Components + +- Handle more complete list of possible git URL forms (ssh:// and ftp:// prefixes specifically) ([#2945](https://github.com/nf-core/tools/pull/2945)) +- Fix path in component update script ([#2823](https://github.com/nf-core/tools/pull/2823)) + +### General + +- Update CI to use nf-core/setup-nextflow v2 ([#2819](https://github.com/nf-core/tools/pull/2819)) +- Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) +- Add `force_pr` flag to sync, to force a PR even though there are no changes committed ([#2822](https://github.com/nf-core/tools/pull/2822)) +- Update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) +- Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)), ([#2902](https://github.com/nf-core/tools/pull/2902)), ([#2927](https://github.com/nf-core/tools/pull/2927)), ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) +- Add tests for assignment and referencing of params in main.nf ([#2841](https://github.com/nf-core/tools/pull/2841)) +- Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) +- Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) +- Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible +- Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) +- Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) +- Ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) +- Fix incorrect assertions for called_with on mocks ([#2891](https://github.com/nf-core/tools/pull/2891)) +- Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) +- Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) +- Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) +- Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) +- Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) +- fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) +- Restructure CHANGELOG.md ([#2954](https://github.com/nf-core/tools/pull/2954)) + ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] ### Template @@ -9,6 +74,7 @@ - Fix topic extraction step for hashtags in toots ([#2810](https://github.com/nf-core/tools/pull/2810)) - Update modules and subworkflows in the template ([#2811](https://github.com/nf-core/tools/pull/2811)) - Unpin setup-nextflow and action-tower-launch ([#2806](https://github.com/nf-core/tools/pull/2806)) +- Add nf-core-version to `.nf-core.yml` ([#2874](https://github.com/nf-core/tools/pull/2874)) ### Download @@ -49,6 +115,8 @@ - Patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) - Handle symlinks when migrating pytest ([#2770](https://github.com/nf-core/tools/pull/2770)) - Add `--profile` parameter to nf-test command ([#2767](https://github.com/nf-core/tools/pull/2767)) +- Reduce the sha length in the `nf-core modules list local` and add links to the specific commit ([#2870](https://github.com/nf-core/tools/pull/2870)) +- Add links the nf-core module page and to open the local file in VSCode for module lint results ([#2870](https://github.com/nf-core/tools/pull/2870)) ### General diff --git a/.github/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 100% rename from .github/CONTRIBUTING.md rename to CONTRIBUTING.md diff --git a/Dockerfile b/Dockerfile index 62431be140..ae3a4e1a3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM python:3.11-slim@sha256:ce81dc539f0aedc9114cae640f8352fad83d37461c24a3615b01f081d0c0583a -LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ - description="Docker image containing requirements for the nfcore tools" +FROM python:3.12-slim@sha256:2be8daddbb82756f7d1f2c7ece706aadcb284bf6ab6d769ea695cc3ed6016743 +LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ + description="Docker image containing requirements for nf-core/tools" # Do not pick up python packages from $HOME ENV PYTHONNUSERSITE=1 diff --git a/README.md b/README.md index cf0b01d210..5222854084 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ conda install nf-core Alternatively, you can create a new environment with both nf-core/tools and nextflow: ```bash -conda create --name nf-core python=3.11 nf-core nextflow +conda create --name nf-core python=3.12 nf-core nextflow conda activate nf-core ``` @@ -376,7 +376,7 @@ The pipeline files are automatically updated (`params.custom_config_base` is set So using `-profile ` should work if available within [nf-core/configs](https://github.com/nf-core/configs). > [!WARNING] -> This option is not available when downloading a pipeline for use with [Nextflow Tower](#adapting-downloads-to-nextflow-tower) because the application manages all configurations separately. +> This option is not available when downloading a pipeline for use with [Seqera Platform](#adapting-downloads-to-seqera-platform) because the application manages all configurations separately. ### Downloading Apptainer containers @@ -428,14 +428,17 @@ Note that compressing many GBs of binary files can be slow, so specifying `--com If the download speeds are much slower than your internet connection is capable of, you can set `--parallel-downloads` to a large number to download loads of images at once. -### Adapting downloads to Nextflow Tower +### Adapting downloads to Seqera Platform -[seqeralabs® Nextflow Tower](https://cloud.tower.nf/) provides a graphical user interface to oversee pipeline runs, gather statistics and configure compute resources. While pipelines added to _Tower_ are preferably hosted at a Git service, providing them as disconnected, self-reliant repositories is also possible for premises with restricted network access. Choosing the `--tower` flag will download the pipeline in an appropriate form. +[Seqera Platform](https://seqera.io/platform/) (formerly _"Nextflow Tower"_) provides a graphical user interface to oversee pipeline runs, gather statistics and configure compute resources. While pipelines added to _Seqera Platform_ are preferably hosted at a Git service, providing them as disconnected, self-reliant repositories is also possible for premises with restricted network access. Choosing the `--platform` flag will download the pipeline in an appropriate form. -Subsequently, the `*.git` folder can be moved to it's final destination and linked with a pipeline in _Tower_ using the `file:/` prefix. +Subsequently, the `*.git` folder can be moved to it's final destination and linked with a pipeline in _Seqera Platform_ using the `file:/` prefix. > [!TIP] -> Also without access to Tower, pipelines downloaded with the `--tower` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. +> Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. + +Facilities and those who are setting up pipelines for others to use may find the `--tag` argument helpful. It allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `--tag "3.12.0=testing" --tag "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. +The `--tag` argument must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. ## Pipeline software licences diff --git a/docs/api/_src/pipeline_lint_tests/base_config.md b/docs/api/_src/pipeline_lint_tests/base_config.md new file mode 100644 index 0000000000..4a56ef9789 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/base_config.md @@ -0,0 +1,5 @@ +# base_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.base_config +``` diff --git a/docs/api/_src/pipeline_lint_tests/modules_config.md b/docs/api/_src/pipeline_lint_tests/modules_config.md new file mode 100644 index 0000000000..2a4f51c5a4 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/modules_config.md @@ -0,0 +1,5 @@ +# modules_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.modules_config +``` diff --git a/docs/api/_src/pipeline_lint_tests/nfcore_yml.md b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md new file mode 100644 index 0000000000..f7e797a29c --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md @@ -0,0 +1,5 @@ +# nfcore_yml + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.nfcore_yml +``` diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt index fc5184b95f..abffe30740 100644 --- a/docs/api/requirements.txt +++ b/docs/api/requirements.txt @@ -1,4 +1,5 @@ Sphinx>=3.3.1 sphinxcontrib-napoleon +sphinx-markdown-builder sphinx_rtd_theme>=0.5.0 myst-parser diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index 70171475d9..8a7ad902f4 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,122 +19,122 @@ font-weight: 700; } - .terminal-3430704182-matrix { + .terminal-3430835254-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3430704182-title { + .terminal-3430835254-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3430704182-r1 { fill: #c5c8c6 } -.terminal-3430704182-r2 { fill: #98a84b } -.terminal-3430704182-r3 { fill: #9a9b99 } -.terminal-3430704182-r4 { fill: #608ab1 } -.terminal-3430704182-r5 { fill: #d0b344 } -.terminal-3430704182-r6 { fill: #cc555a } + .terminal-3430835254-r1 { fill: #c5c8c6 } +.terminal-3430835254-r2 { fill: #98a84b } +.terminal-3430835254-r3 { fill: #9a9b99 } +.terminal-3430835254-r4 { fill: #608ab1 } +.terminal-3430835254-r5 { fill: #d0b344 } +.terminal-3430835254-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -146,41 +146,41 @@ - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1' -INFO     Updated version in 'nextflow.config' - - version         = '1.0dev' - + version         = '1.1' - - -INFO     Updated version in 'assets/multiqc_config.yml' - - This report has been generated by the <a  -href="https://github.com/nf-core/nextbigthing/tree/dev" target="_blank">nf-core/nextbigthing</a> - + This report has been generated by the <a  -href="https://github.com/nf-core/nextbigthing/releases/tag/1.1"  -target="_blank">nf-core/nextbigthing</a> - - -INFO     Updated version in 'assets/multiqc_config.yml' - - <a href="https://nf-co.re/nextbigthing/dev/docs/output"  -target="_blank">documentation</a>. - + <a href="https://nf-co.re/nextbigthing/1.1/docs/output"  -target="_blank">documentation</a>. - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1' +INFO     Updated version in 'nextflow.config' + - version         = '1.0dev' + + version         = '1.1' + + +INFO     Updated version in 'assets/multiqc_config.yml' + - This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/tree/dev" target="_blank">nf-core/nextbigthing</a> + + This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/releases/tag/1.1"  +target="_blank">nf-core/nextbigthing</a> + + +INFO     Updated version in 'assets/multiqc_config.yml' + - <a href="https://nf-co.re/nextbigthing/dev/docs/output"  +target="_blank">documentation</a>. + + <a href="https://nf-co.re/nextbigthing/1.1/docs/output"  +target="_blank">documentation</a>. + + diff --git a/docs/images/nf-core-create-logo.svg b/docs/images/nf-core-create-logo.svg index c6e7269363..50340ff44f 100644 --- a/docs/images/nf-core-create-logo.svg +++ b/docs/images/nf-core-create-logo.svg @@ -19,62 +19,62 @@ font-weight: 700; } - .terminal-2584676837-matrix { + .terminal-2584807909-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2584676837-title { + .terminal-2584807909-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2584676837-r1 { fill: #c5c8c6 } -.terminal-2584676837-r2 { fill: #98a84b } -.terminal-2584676837-r3 { fill: #9a9b99 } -.terminal-2584676837-r4 { fill: #608ab1 } -.terminal-2584676837-r5 { fill: #d0b344 } -.terminal-2584676837-r6 { fill: #98729f } + .terminal-2584807909-r1 { fill: #c5c8c6 } +.terminal-2584807909-r2 { fill: #98a84b } +.terminal-2584807909-r3 { fill: #9a9b99 } +.terminal-2584807909-r4 { fill: #608ab1 } +.terminal-2584807909-r5 { fill: #d0b344 } +.terminal-2584807909-r6 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + @@ -86,21 +86,21 @@ - + - - $ nf-core create-logo nextbigthing - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Created logo: nf-core-nextbigthing_logo_light.png + + $ nf-core create-logo nextbigthing + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Created logo: nf-core-nextbigthing_logo_light.png diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 422e6734ce..70b281cd1c 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,104 +19,104 @@ font-weight: 700; } - .terminal-165925560-matrix { + .terminal-166056632-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-165925560-title { + .terminal-166056632-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-165925560-r1 { fill: #c5c8c6 } -.terminal-165925560-r2 { fill: #98a84b } -.terminal-165925560-r3 { fill: #9a9b99 } -.terminal-165925560-r4 { fill: #608ab1 } -.terminal-165925560-r5 { fill: #d0b344 } -.terminal-165925560-r6 { fill: #98729f } -.terminal-165925560-r7 { fill: #ff2c7a } -.terminal-165925560-r8 { fill: #98a84b;font-weight: bold } -.terminal-165925560-r9 { fill: #1984e9;text-decoration: underline; } + .terminal-166056632-r1 { fill: #c5c8c6 } +.terminal-166056632-r2 { fill: #98a84b } +.terminal-166056632-r3 { fill: #9a9b99 } +.terminal-166056632-r4 { fill: #608ab1 } +.terminal-166056632-r5 { fill: #d0b344 } +.terminal-166056632-r6 { fill: #98729f } +.terminal-166056632-r7 { fill: #ff2c7a } +.terminal-166056632-r8 { fill: #98a84b;font-weight: bold } +.terminal-166056632-r9 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -128,34 +128,34 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' -INFO     Initialising pipeline git repository                                                        -INFO     Done. Remember to add a remote and push to GitHub:                                          - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      -INFO    !!!!!! IMPORTANT !!!!!! - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' +INFO     Initialising pipeline git repository                                                        +INFO     Done. Remember to add a remote and push to GitHub:                                          + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      +INFO    !!!!!! IMPORTANT !!!!!! + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 5594930fa7..e7511c8f54 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,86 +19,86 @@ font-weight: 700; } - .terminal-2899766736-matrix { + .terminal-2899897808-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2899766736-title { + .terminal-2899897808-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2899766736-r1 { fill: #c5c8c6 } -.terminal-2899766736-r2 { fill: #98a84b } -.terminal-2899766736-r3 { fill: #9a9b99 } -.terminal-2899766736-r4 { fill: #608ab1 } -.terminal-2899766736-r5 { fill: #d0b344 } -.terminal-2899766736-r6 { fill: #cc555a } + .terminal-2899897808-r1 { fill: #c5c8c6 } +.terminal-2899897808-r2 { fill: #98a84b } +.terminal-2899897808-r3 { fill: #9a9b99 } +.terminal-2899897808-r4 { fill: #608ab1 } +.terminal-2899897808-r5 { fill: #d0b344 } +.terminal-2899897808-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -110,29 +110,29 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -WARNING  Could not find GitHub authentication token. Some API requests may fail.                     -INFO     Saving 'nf-core/rnaseq' -          Pipeline revision: '3.8' -          Use containers: 'none' -          Container library: 'quay.io' -          Output directory: 'nf-core-rnaseq' -          Include default institutional configuration: 'True' -INFO     Downloading centralised configs from GitHub                                                 -INFO     Downloading workflow files from GitHub                                                      + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +WARNING  Could not find GitHub authentication token. Some API requests may fail.                     +INFO     Saving 'nf-core/rnaseq' +          Pipeline revision: '3.8' +          Use containers: 'none' +          Container library: 'quay.io' +          Output directory: 'nf-core-rnaseq' +          Include default institutional configuration: 'True' +INFO     Downloading centralised configs from GitHub                                                 +INFO     Downloading workflow files from GitHub                                                      diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index f2608fe76f..ef0cfc2ba8 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-3860049459-matrix { + .terminal-3860180531-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3860049459-title { + .terminal-3860180531-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3860049459-r1 { fill: #c5c8c6 } -.terminal-3860049459-r2 { fill: #98a84b } -.terminal-3860049459-r3 { fill: #9a9b99 } -.terminal-3860049459-r4 { fill: #608ab1 } -.terminal-3860049459-r5 { fill: #d0b344 } -.terminal-3860049459-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3860049459-r7 { fill: #68a0b3;font-weight: bold } + .terminal-3860180531-r1 { fill: #c5c8c6 } +.terminal-3860180531-r2 { fill: #98a84b } +.terminal-3860180531-r3 { fill: #9a9b99 } +.terminal-3860180531-r4 { fill: #608ab1 } +.terminal-3860180531-r5 { fill: #d0b344 } +.terminal-3860180531-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3860180531-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      -         files or profiles                                                                           - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1) + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      +         files or profiles                                                                           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 8cc00c351b..d8aa9b03a3 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,62 +19,62 @@ font-weight: 700; } - .terminal-918620543-matrix { + .terminal-918751615-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-918620543-title { + .terminal-918751615-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-918620543-r1 { fill: #c5c8c6 } -.terminal-918620543-r2 { fill: #98a84b } -.terminal-918620543-r3 { fill: #9a9b99 } -.terminal-918620543-r4 { fill: #608ab1 } -.terminal-918620543-r5 { fill: #d0b344 } -.terminal-918620543-r6 { fill: #68a0b3;font-weight: bold } + .terminal-918751615-r1 { fill: #c5c8c6 } +.terminal-918751615-r2 { fill: #98a84b } +.terminal-918751615-r3 { fill: #9a9b99 } +.terminal-918751615-r4 { fill: #608ab1 } +.terminal-918751615-r5 { fill: #d0b344 } +.terminal-918751615-r6 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + @@ -86,21 +86,21 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 2e55a7e11e..9276638e99 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ -││ -│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ -│doi (after the first release).                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - - -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔] 188 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Testing pipeline: . + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/requests/models.py:971 in    │ +│json│ +││ +│ 968 │   │   │   │   │   raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)                    │ +│ 969 │   │   │ +│ 970 │   │   try:                                                                              │ +│❱  971 │   │   │   return complexjson.loads(self.text, **kwargs)                                 │ +│ 972 │   │   except JSONDecodeError as e:                                                      │ +│ 973 │   │   │   # Catch JSON-related errors and raise as requests.JSONDecodeError│ +│ 974 │   │   │   # This aliases json.JSONDecodeError and simplejson.JSONDecodeError│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/json/__init__.py:346 in loads│ +││ +│343 │   if (clsisNoneand object_hook isNoneand│ +│344 │   │   │   parse_int isNoneand parse_float isNoneand│ +│345 │   │   │   parse_constant isNoneand object_pairs_hook isNoneandnot kw):              │ +│❱ 346 │   │   return _default_decoder.decode(s)                                                  │ +│347 │   ifclsisNone:                                                                        │ +│348 │   │   cls = JSONDecoder                                                                  │ +│349 │   if object_hook isnotNone:                                                            │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/json/decoder.py:337 in decode│ +││ +│334 │   │   containing a JSON document).│ +│335 │   │   │ +│336 │   │   """│ +│❱ 337 │   │   obj, end = self.raw_decode(s, idx=_w(s, 0).end())                                  │ +│338 │   │   end = _w(s, end).end()                                                             │ +│339 │   │   if end != len(s):                                                                  │ +│340 │   │   │   raise JSONDecodeError("Extra data", s, end)                                    │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/json/decoder.py:355 in raw_decode│ +││ +│352 │   │   try:                                                                               │ +│353 │   │   │   obj, end = self.scan_once(s, idx)                                              │ +│354 │   │   exceptStopIterationas err:                                                       │ +│❱ 355 │   │   │   raise JSONDecodeError("Expecting value", s, err.value) fromNone│ +│356 │   │   return obj, end                                                                    │ +│357 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +JSONDecodeError: Expecting value: line 1 column 1(char 0) + +During handling of the above exception, another exception occurred: + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.12.3/x64/bin/nf-core:8 in <module>│ +││ +│5 fromnf_core.__main__import run_nf_core                                                     │ +│6 if__name__ == '__main__':                                                                   │ +│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ +│❱ 8 │   sys.exit(run_nf_core())                                                                  │ +│9 │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/__main__.py:150 in   │ +│run_nf_core│ +││ +│ 147 │   │   │   log.debug(f"Could not check latest version: {e}")                             │ +│ 148 │   │   stderr.print("\n")                                                                │ +│ 149 │   # Launch the click cli│ +│❱  150 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ +│ 151 │ +│ 152 │ +│ 153 @tui()                                                                                    │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/rich_click/rich_command.py:3│ +│67 in __call__│ +││ +│364 │   │   # Include this here because I run into a false warning│ +│365 │   │   # in the PyCharm IDE otherwise; for some reason PyCharm doesn't│ +│366 │   │   # seem to think RichGroups are callable. (No issues with Mypy, though.)│ +│❱ 367 │   │   returnsuper().__call__(*args, **kwargs)                                           │ +│368 │ +│369 │ +│370 classRichCommandCollection(CommandCollection, RichGroup):                                 │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:1157 in        │ +│__call__│ +││ +│1154 │   │ +│1155 │   def__call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:                           │ +│1156 │   │   """Alias for :meth:`main`."""│ +│❱ 1157 │   │   returnself.main(*args, **kwargs)                                                 │ +│1158 │ +│1159 │ +│1160 classCommand(BaseCommand):                                                               │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/rich_click/rich_command.py:1│ +│52 in main│ +││ +│149 │   │   try:                                                                               │ +│150 │   │   │   try:                                                                           │ +│151 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                   │ +│❱ 152 │   │   │   │   │   rv = self.invoke(ctx)                                                  │ +│153 │   │   │   │   │   ifnot standalone_mode:                                                │ +│154 │   │   │   │   │   │   return rv                                                          │ +│155 │   │   │   │   │   # it's not safe to `ctx.exit(rv)` here!│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:1688 in invoke│ +││ +│1685 │   │   │   │   super().invoke(ctx)                                                       │ +│1686 │   │   │   │   sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)                    │ +│1687 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1688 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1689 │   │   │ +│1690 │   │   # In chain mode we create the contexts step by step, but after the│ +│1691 │   │   # base command has been invoked.  Because at that point we do not│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:1434 in invoke│ +││ +│1431 │   │   │   echo(style(message, fg="red"), err=True)                                      │ +│1432 │   │   │ +│1433 │   │   ifself.callback isnotNone:                                                     │ +│❱ 1434 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ +│1435 │   │ +│1436 │   defshell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]:  │ +│1437 │   │   """Return a list of completions for the incomplete value. Looks│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:783 in invoke│ +││ +│ 780 │   │   │ +│ 781 │   │   with augment_usage_errors(__self):                                                │ +│ 782 │   │   │   with ctx:                                                                     │ +│❱  783 │   │   │   │   return __callback(*args, **kwargs)                                        │ +│ 784 │   │ +│ 785 │   defforward(                                                                          │ +│ 786 │   │   __self, __cmd: "Command", *args: t.Any, **kwargs: t.Any  # noqa: B902│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/decorators.py:33 in    │ +│new_func│ +││ +│ 30 │   """│ +│ 31 │   │ +│ 32 │   defnew_func(*args: "P.args", **kwargs: "P.kwargs") -> "R":                            │ +│❱  33 │   │   return f(get_current_context(), *args, **kwargs)                                   │ +│ 34 │   │ +│ 35 │   return update_wrapper(new_func, f)                                                     │ +│ 36 │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/__main__.py:635 in   │ +│lint│ +││ +│ 632 │   │ +│ 633 │   # Run the lint tests!│ +│ 634 │   try:                                                                                  │ +│❱  635 │   │   lint_obj, module_lint_obj, subworkflow_lint_obj = run_linting(                    │ +│ 636 │   │   │   dir,                                                                          │ +│ 637 │   │   │   release,                                                                      │ +│ 638 │   │   │   fix,                                                                          │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/lint/__init__.py:603│ +│ in run_linting│ +││ +│600 │   │ +│601 │   # Run the pipeline linting tests│ +│602 │   try:                                                                                   │ +│❱ 603 │   │   lint_obj._lint_pipeline()                                                          │ +│604 │   exceptAssertionErroras e:                                                            │ +│605 │   │   log.critical(f"Critical error: {e}")                                               │ +│606 │   │   log.info("Stopping tests...")                                                      │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/lint/__init__.py:228│ +│ in _lint_pipeline│ +││ +│225 │   │   │   │   │   continue│ +│226 │   │   │   │   self.progress_bar.update(lint_progress, advance=1, test_name=test_name)    │ +│227 │   │   │   │   log.debug(f"Running lint test: {test_name}")                               │ +│❱ 228 │   │   │   │   test_results = getattr(self, test_name)()                                  │ +│229 │   │   │   │   for test in test_results.get("passed", []):                                │ +│230 │   │   │   │   │   self.passed.append((test_name, test))                                  │ +│231 │   │   │   │   for test in test_results.get("ignored", []):                               │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/lint/actions_schema_│ +│validation.py:31 in actions_schema_validation│ +││ +│28 │   │ +│29 │   # Load the GitHub workflow schema│ +│30 │   r = requests.get("https://json.schemastore.org/github-workflow", allow_redirects=Tru│ +│❱ 31 │   schema = r.json()                                                                       │ +│32 │   │ +│33 │   # Validate all workflows against the schema│ +│34 │   for wf_path in action_workflows:                                                        │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/requests/models.py:975 in    │ +│json│ +││ +│ 972 │   │   except JSONDecodeError as e:                                                      │ +│ 973 │   │   │   # Catch JSON-related errors and raise as requests.JSONDecodeError│ +│ 974 │   │   │   # This aliases json.JSONDecodeError and simplejson.JSONDecodeError│ +│❱  975 │   │   │   raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)                            │ +│ 976 │   │ +│ 977 │   @property│ +│ 978 │   deflinks(self):                                                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +JSONDecodeError: Expecting value: line 1 column 1(char 0) diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 643545c6fc..26d224ddc2 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   742 │         3.14.0 │ 2 months ago │           - │ -                   │ -│ differentialabundan… │    38 │          1.4.0 │ 3 months ago │           - │ -                   │ -│ smrnaseq             │    64 │          2.3.0 │   6 days ago │           - │ -                   │ -│ rnasplice            │    25 │          1.0.3 │   6 days ago │           - │ -                   │ -│ circrna              │    34 │            dev │  1 weeks ago │           - │ -                   │ -│ scrnaseq             │   125 │          2.5.1 │ 1 months ago │           - │ -                   │ -│ rnafusion            │   126 │          3.0.1 │ 3 months ago │           - │ -                   │ -│ spatialtranscriptom… │    36 │            dev │  4 weeks ago │           - │ -                   │ -│ dualrnaseq           │    16 │          1.0.0 │  3 years ago │           - │ -                   │ -│ marsseq              │     5 │          1.0.3 │ 8 months ago │           - │ -                   │ -│ lncpipe              │    28 │            dev │  1 years ago │           - │ -                   │ -│ scflow               │    24 │            dev │  3 years ago │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ differentialabunda… │    45 │          1.4.0 │  5 months ago │           - │ -                   │ +│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ +│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ +│ spatialvi           │    39 │            dev │     yesterday │           - │ -                   │ +│ scnanoseq           │     5 │            dev │    1 week ago │           - │ -                   │ +│ scrnaseq            │   169 │          2.6.0 │  18 hours ago │           - │ -                   │ +│ circrna             │    40 │            dev │   2 weeks ago │           - │ -                   │ +│ rnafusion           │   129 │          3.0.2 │   4 weeks ago │           - │ -                   │ +│ rnasplice           │    33 │          1.0.3 │  2 months ago │           - │ -                   │ +│ dualrnaseq          │    16 │          1.0.0 │   3 years ago │           - │ -                   │ +│ marsseq             │     5 │          1.0.3 │ 10 months ago │           - │ -                   │ +│ lncpipe             │    30 │            dev │   2 years ago │           - │ -                   │ +│ scflow              │    24 │            dev │   3 years ago │           - │ -                   │ +└─────────────────────┴───────┴────────────────┴───────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 8ea120599f..be2b84287b 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-3700160896-matrix { + .terminal-358676910-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3700160896-title { + .terminal-358676910-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3700160896-r1 { fill: #c5c8c6 } -.terminal-3700160896-r2 { fill: #98a84b } -.terminal-3700160896-r3 { fill: #9a9b99 } -.terminal-3700160896-r4 { fill: #608ab1 } -.terminal-3700160896-r5 { fill: #d0b344 } -.terminal-3700160896-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3700160896-r7 { fill: #868887 } -.terminal-3700160896-r8 { fill: #868887;font-style: italic; } + .terminal-358676910-r1 { fill: #c5c8c6 } +.terminal-358676910-r2 { fill: #98a84b } +.terminal-358676910-r3 { fill: #9a9b99 } +.terminal-358676910-r4 { fill: #608ab1 } +.terminal-358676910-r5 { fill: #d0b344 } +.terminal-358676910-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-358676910-r7 { fill: #868887 } +.terminal-358676910-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq              │   742 │         3.14.0 │  2 months ago │           - │ -                   │ -│ sarek               │   320 │          3.4.0 │  4 months ago │           - │ -                   │ -│ mag                 │   175 │          2.5.4 │   2 weeks ago │           - │ -                   │ -│ chipseq             │   161 │          2.0.0 │   1 years ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │  20 hours ago │           - │ -                   │ +│ mag                 │   182 │          2.5.4 │  3 months ago │           - │ -                   │ +│ chipseq             │   172 │          2.0.0 │   2 years ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 5e4939f746..6857db79f9 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3505815442-matrix { + .terminal-4024274838-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3505815442-title { + .terminal-4024274838-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3505815442-r1 { fill: #c5c8c6 } -.terminal-3505815442-r2 { fill: #98a84b } -.terminal-3505815442-r3 { fill: #9a9b99 } -.terminal-3505815442-r4 { fill: #608ab1 } -.terminal-3505815442-r5 { fill: #d0b344 } -.terminal-3505815442-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3505815442-r7 { fill: #868887 } -.terminal-3505815442-r8 { fill: #868887;font-style: italic; } + .terminal-4024274838-r1 { fill: #c5c8c6 } +.terminal-4024274838-r2 { fill: #98a84b } +.terminal-4024274838-r3 { fill: #9a9b99 } +.terminal-4024274838-r4 { fill: #608ab1 } +.terminal-4024274838-r5 { fill: #d0b344 } +.terminal-4024274838-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4024274838-r7 { fill: #868887 } +.terminal-4024274838-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ raredisease         │    65 │          1.1.1 │  7 months ago │           - │ -                   │ -│ fetchngs            │   101 │         1.11.0 │  4 months ago │           - │ -                   │ -│ sarek               │   320 │          3.4.0 │  4 months ago │           - │ -                   │ -│ methylseq           │   126 │          2.6.0 │  2 months ago │           - │ -                   │ -│ rnaseq              │   742 │         3.14.0 │  2 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ differentialabunda… │    45 │          1.4.0 │  5 months ago │           - │ -                   │ +│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │  20 hours ago │           - │ -                   │ +│ variantbenchmarking │     4 │            dev │  19 hours ago │           - │ -                   │ +│ meerpipe            │     0 │            dev │     yesterday │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 093a0cebc6..4c342507a3 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1669068868-matrix { + .terminal-1669199940-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1669068868-title { + .terminal-1669199940-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1669068868-r1 { fill: #c5c8c6 } -.terminal-1669068868-r2 { fill: #98a84b } -.terminal-1669068868-r3 { fill: #9a9b99 } -.terminal-1669068868-r4 { fill: #608ab1 } -.terminal-1669068868-r5 { fill: #d0b344 } -.terminal-1669068868-r6 { fill: #98a84b;font-weight: bold } -.terminal-1669068868-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-1669199940-r1 { fill: #c5c8c6 } +.terminal-1669199940-r2 { fill: #98a84b } +.terminal-1669199940-r3 { fill: #9a9b99 } +.terminal-1669199940-r4 { fill: #608ab1 } +.terminal-1669199940-r5 { fill: #d0b344 } +.terminal-1669199940-r6 { fill: #98a84b;font-weight: bold } +.terminal-1669199940-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index caa9c389f0..6cd7e15b66 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' +INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' +INFO     Using Singularity container:                                                                +'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' +INFO     Created component template: 'fastqc' +INFO     Created following files:                                                                    +           modules/nf-core/fastqc/main.nf                                                            +           modules/nf-core/fastqc/meta.yml                                                           +           modules/nf-core/fastqc/environment.yml                                                    +           modules/nf-core/fastqc/tests/tags.yml                                                     +           modules/nf-core/fastqc/tests/main.nf.test                                                 diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 255c595e7c..92d19331ab 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-1309602353-matrix { + .terminal-1309733425-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1309602353-title { + .terminal-1309733425-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1309602353-r1 { fill: #c5c8c6 } -.terminal-1309602353-r2 { fill: #98a84b } -.terminal-1309602353-r3 { fill: #9a9b99 } -.terminal-1309602353-r4 { fill: #608ab1 } -.terminal-1309602353-r5 { fill: #d0b344 } -.terminal-1309602353-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1309602353-r7 { fill: #98a84b;font-weight: bold } -.terminal-1309602353-r8 { fill: #868887 } -.terminal-1309602353-r9 { fill: #d08442 } -.terminal-1309602353-r10 { fill: #868887;font-style: italic; } -.terminal-1309602353-r11 { fill: #98729f } + .terminal-1309733425-r1 { fill: #c5c8c6 } +.terminal-1309733425-r2 { fill: #98a84b } +.terminal-1309733425-r3 { fill: #9a9b99 } +.terminal-1309733425-r4 { fill: #608ab1 } +.terminal-1309733425-r5 { fill: #d0b344 } +.terminal-1309733425-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1309733425-r7 { fill: #98a84b;font-weight: bold } +.terminal-1309733425-r8 { fill: #868887 } +.terminal-1309733425-r9 { fill: #d08442 } +.terminal-1309733425-r10 { fill: #868887;font-style: italic; } +.terminal-1309733425-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (file) │List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (file) │List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 2b0331bcaa..51487ef6a5 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-310938767-matrix { + .terminal-311069839-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-310938767-title { + .terminal-311069839-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-310938767-r1 { fill: #c5c8c6 } -.terminal-310938767-r2 { fill: #98a84b } -.terminal-310938767-r3 { fill: #9a9b99 } -.terminal-310938767-r4 { fill: #608ab1 } -.terminal-310938767-r5 { fill: #d0b344 } + .terminal-311069839-r1 { fill: #c5c8c6 } +.terminal-311069839-r2 { fill: #98a84b } +.terminal-311069839-r3 { fill: #9a9b99 } +.terminal-311069839-r4 { fill: #608ab1 } +.terminal-311069839-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Installing 'abacas' -INFO     Use the following statement to include this module:                                         - - include { ABACAS } from '../modules/nf-core/abacas/main'                                            - + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 2809eeee22..355ca8d32d 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -19,67 +19,67 @@ font-weight: 700; } - .terminal-1680431948-matrix { + .terminal-1680563020-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1680431948-title { + .terminal-1680563020-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1680431948-r1 { fill: #c5c8c6 } -.terminal-1680431948-r2 { fill: #98a84b } -.terminal-1680431948-r3 { fill: #9a9b99 } -.terminal-1680431948-r4 { fill: #608ab1 } -.terminal-1680431948-r5 { fill: #d0b344 } + .terminal-1680563020-r1 { fill: #c5c8c6 } +.terminal-1680563020-r2 { fill: #98a84b } +.terminal-1680563020-r3 { fill: #9a9b99 } +.terminal-1680563020-r4 { fill: #608ab1 } +.terminal-1680563020-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -91,23 +91,23 @@ - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index ecb9ed499a..4956396bcf 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - + - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -self.repo_type='pipeline' -┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name┃Repository           ┃Version SHA          ┃Message              ┃Date      ┃ -┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ fastqc      │ https://github.com/n… │ f4ae1d942bd50c5c0b9b… │ Update FASTQC to use  │ 2024-01-31 │ -│             │                       │                       │ unique names for      │            │ -│             │                       │                       │ snapshots (#4825)     │            │ -│ multiqc     │ https://github.com/n… │ b7ebe95761cd389603f9… │ Update MQC container  │ 2024-02-29 │ -│             │                       │                       │ (#5006)               │            │ -└─────────────┴───────────────────────┴───────────────────────┴───────────────────────┴────────────┘ + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: pipeline +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name┃Repository     ┃Version SHA┃Message                             ┃Date      ┃ +┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ fastqc      │ nf-core/modules │ 285a505     │ Fix FastQC memory allocation (#5432) │ 2024-04-05 │ +│ multiqc     │ nf-core/modules │ b7ebe95     │ Update MQC container (#5006)         │ 2024-02-29 │ +└─────────────┴─────────────────┴─────────────┴──────────────────────────────────────┴────────────┘ diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 138be73068..b6e6fb3a8e 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2384550328-matrix { + .terminal-2384681400-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2384550328-title { + .terminal-2384681400-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2384550328-r1 { fill: #c5c8c6 } -.terminal-2384550328-r2 { fill: #98a84b } -.terminal-2384550328-r3 { fill: #9a9b99 } -.terminal-2384550328-r4 { fill: #608ab1 } -.terminal-2384550328-r5 { fill: #d0b344 } -.terminal-2384550328-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-2384550328-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2384550328-r8 { fill: #868887;font-style: italic; } + .terminal-2384681400-r1 { fill: #c5c8c6 } +.terminal-2384681400-r2 { fill: #98a84b } +.terminal-2384681400-r3 { fill: #9a9b99 } +.terminal-2384681400-r4 { fill: #608ab1 } +.terminal-2384681400-r5 { fill: #d0b344 } +.terminal-2384681400-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-2384681400-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2384681400-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):                     - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                                          ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                                │ -│ abricate/run                                          │ -│ abricate/summary                                      │ -│ abritamr/run                                          │ -│ adapterremoval                                        │ -│ adapterremovalfixprefix                               │ -│ admixture                                             │ -│ affy/justrma                                          │ -│ agat/convertspgff2gtf                                 │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ abritamr/run                                          │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ admixture                                             │ +│ affy/justrma                                          │ +│ agat/convertspgff2gtf                                 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 5ed5e2f4d1..34d9c8dfb7 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,127 +19,127 @@ font-weight: 700; } - .terminal-1838296137-matrix { + .terminal-2649828425-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1838296137-title { + .terminal-2649828425-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1838296137-r1 { fill: #c5c8c6 } -.terminal-1838296137-r2 { fill: #98a84b } -.terminal-1838296137-r3 { fill: #9a9b99 } -.terminal-1838296137-r4 { fill: #608ab1 } -.terminal-1838296137-r5 { fill: #d0b344 } -.terminal-1838296137-r6 { fill: #ff2627 } -.terminal-1838296137-r7 { fill: #00823d } -.terminal-1838296137-r8 { fill: #ff2c7a;font-weight: bold } + .terminal-2649828425-r1 { fill: #c5c8c6 } +.terminal-2649828425-r2 { fill: #98a84b } +.terminal-2649828425-r3 { fill: #9a9b99 } +.terminal-2649828425-r4 { fill: #608ab1 } +.terminal-2649828425-r5 { fill: #d0b344 } +.terminal-2649828425-r6 { fill: #ff2627 } +.terminal-2649828425-r7 { fill: #00823d } +.terminal-2649828425-r8 { fill: #ff2c7a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -151,42 +151,42 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Changes in module 'nf-core/fastqc' -INFO    'modules/nf-core/fastqc/environment.yml' is unchanged                                       -INFO    'modules/nf-core/fastqc/meta.yml' is unchanged                                              -INFO     Changes in 'fastqc/main.nf':                                                                - ---- modules/nf-core/fastqc/main.nf -+++ modules/nf-core/fastqc/main.nf -@@ -1,6 +1,6 @@ -process FASTQC {                                                                                   -    tag "$meta.id"                                                                                 --    label 'process_medium' -+    label 'process_low' - -    conda "${moduleDir}/environment.yml"                                                           -    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  - - -INFO    'modules/nf-core/fastqc/tests/tags.yml' is unchanged                                        -INFO    'modules/nf-core/fastqc/tests/main.nf.test' is unchanged                                    -INFO    'modules/nf-core/fastqc/tests/main.nf.test.snap' is unchanged                               -INFO     Patch file of 'modules/nf-core/fastqc' written to 'modules/nf-core/fastqc/fastqc.diff' + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Changes in module 'nf-core/fastqc' +INFO     Changes in 'fastqc/main.nf':                                                                + +--- modules/nf-core/fastqc/main.nf ++++ modules/nf-core/fastqc/main.nf +@@ -1,6 +1,6 @@ +process FASTQC {                                                                                   +    tag "$meta.id"                                                                                 +-    label 'process_medium' ++    label 'process_low' + +    conda "${moduleDir}/environment.yml"                                                           +    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  + + +INFO    'modules/nf-core/fastqc/meta.yml' is unchanged                                              +INFO    'modules/nf-core/fastqc/environment.yml' is unchanged                                       +INFO    'modules/nf-core/fastqc/tests/tags.yml' is unchanged                                        +INFO    'modules/nf-core/fastqc/tests/main.nf.test.snap' is unchanged                               +INFO    'modules/nf-core/fastqc/tests/main.nf.test' is unchanged                                    +INFO     Patch file of 'modules/nf-core/fastqc' written to 'modules/nf-core/fastqc/fastqc.diff' diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 91f7020043..af35fe2d31 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-2115907041-matrix { + .terminal-2116038113-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2115907041-title { + .terminal-2116038113-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2115907041-r1 { fill: #c5c8c6 } -.terminal-2115907041-r2 { fill: #98a84b } -.terminal-2115907041-r3 { fill: #9a9b99 } -.terminal-2115907041-r4 { fill: #608ab1 } -.terminal-2115907041-r5 { fill: #d0b344 } + .terminal-2116038113-r1 { fill: #c5c8c6 } +.terminal-2116038113-r2 { fill: #98a84b } +.terminal-2116038113-r3 { fill: #9a9b99 } +.terminal-2116038113-r4 { fill: #608ab1 } +.terminal-2116038113-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Removed files for 'abacas' and its dependencies 'abacas'.                                   + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and its dependencies 'abacas'.                                   diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 52e86ba826..cb1304e879 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-9171199-matrix { + .terminal-9302271-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-9171199-title { + .terminal-9302271-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-9171199-r1 { fill: #c5c8c6 } -.terminal-9171199-r2 { fill: #98a84b } -.terminal-9171199-r3 { fill: #9a9b99 } -.terminal-9171199-r4 { fill: #608ab1 } -.terminal-9171199-r5 { fill: #d0b344 } + .terminal-9302271-r1 { fill: #c5c8c6 } +.terminal-9302271-r2 { fill: #98a84b } +.terminal-9302271-r3 { fill: #9a9b99 } +.terminal-9302271-r4 { fill: #608ab1 } +.terminal-9302271-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules test fastqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Generating nf-test snapshot                                                                 + + $ nf-core modules test fastqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Generating nf-test snapshot                                                                 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 23d9931ce2..46983bf5ea 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,73 +19,73 @@ font-weight: 700; } - .terminal-2984179218-matrix { + .terminal-2984310290-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2984179218-title { + .terminal-2984310290-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2984179218-r1 { fill: #c5c8c6 } -.terminal-2984179218-r2 { fill: #98a84b } -.terminal-2984179218-r3 { fill: #9a9b99 } -.terminal-2984179218-r4 { fill: #608ab1 } -.terminal-2984179218-r5 { fill: #d0b344 } + .terminal-2984310290-r1 { fill: #c5c8c6 } +.terminal-2984310290-r2 { fill: #98a84b } +.terminal-2984310290-r3 { fill: #9a9b99 } +.terminal-2984310290-r4 { fill: #608ab1 } +.terminal-2984310290-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,25 +97,25 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO    'modules/nf-core/fastqc' is already up to date                                              -INFO    'modules/nf-core/multiqc' is already up to date                                             -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO    'modules/nf-core/fastqc' is already up to date                                              +INFO    'modules/nf-core/multiqc' is already up to date                                             +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 7236440a0d..6d3e448b04 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-978569984-matrix { + .terminal-1038338818-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-978569984-title { + .terminal-1038338818-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-978569984-r1 { fill: #c5c8c6 } -.terminal-978569984-r2 { fill: #98a84b } -.terminal-978569984-r3 { fill: #9a9b99 } -.terminal-978569984-r4 { fill: #608ab1 } -.terminal-978569984-r5 { fill: #d0b344 } -.terminal-978569984-r6 { fill: #98a84b;font-weight: bold } -.terminal-978569984-r7 { fill: #868887;font-weight: bold } -.terminal-978569984-r8 { fill: #868887 } -.terminal-978569984-r9 { fill: #4e707b;font-weight: bold } -.terminal-978569984-r10 { fill: #68a0b3;font-weight: bold } + .terminal-1038338818-r1 { fill: #c5c8c6 } +.terminal-1038338818-r2 { fill: #98a84b } +.terminal-1038338818-r3 { fill: #9a9b99 } +.terminal-1038338818-r4 { fill: #608ab1 } +.terminal-1038338818-r5 { fill: #d0b344 } +.terminal-1038338818-r6 { fill: #98a84b;font-weight: bold } +.terminal-1038338818-r7 { fill: #868887;font-weight: bold } +.terminal-1038338818-r8 { fill: #868887 } +.terminal-1038338818-r9 { fill: #4e707b;font-weight: bold } +.terminal-1038338818-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 30 params) -INFO     Writing schema with 31 params: 'nextflow_schema.json' + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) +INFO     Writing schema with 32 params: 'nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index d08a01144b..1b1a836556 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-3566947207-matrix { + .terminal-3582544776-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3566947207-title { + .terminal-3582544776-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3566947207-r1 { fill: #c5c8c6 } -.terminal-3566947207-r2 { fill: #98a84b } -.terminal-3566947207-r3 { fill: #9a9b99 } -.terminal-3566947207-r4 { fill: #608ab1 } -.terminal-3566947207-r5 { fill: #d0b344 } -.terminal-3566947207-r6 { fill: #98a84b;font-weight: bold } -.terminal-3566947207-r7 { fill: #868887;font-weight: bold } -.terminal-3566947207-r8 { fill: #868887 } -.terminal-3566947207-r9 { fill: #4e707b;font-weight: bold } + .terminal-3582544776-r1 { fill: #c5c8c6 } +.terminal-3582544776-r2 { fill: #98a84b } +.terminal-3582544776-r3 { fill: #9a9b99 } +.terminal-3582544776-r4 { fill: #608ab1 } +.terminal-3582544776-r5 { fill: #d0b344 } +.terminal-3582544776-r6 { fill: #98a84b;font-weight: bold } +.terminal-3582544776-r7 { fill: #868887;font-weight: bold } +.terminal-3582544776-r8 { fill: #868887 } +.terminal-3582544776-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 31 params) + + $ nf-core schema lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 32 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 33984ccbeb..94663e3812 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-2456222967-matrix { + .terminal-2456354039-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2456222967-title { + .terminal-2456354039-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2456222967-r1 { fill: #c5c8c6 } -.terminal-2456222967-r2 { fill: #98a84b } -.terminal-2456222967-r3 { fill: #9a9b99 } -.terminal-2456222967-r4 { fill: #608ab1 } -.terminal-2456222967-r5 { fill: #d0b344 } -.terminal-2456222967-r6 { fill: #98a84b;font-weight: bold } -.terminal-2456222967-r7 { fill: #868887;font-weight: bold } -.terminal-2456222967-r8 { fill: #868887 } -.terminal-2456222967-r9 { fill: #4e707b;font-weight: bold } + .terminal-2456354039-r1 { fill: #c5c8c6 } +.terminal-2456354039-r2 { fill: #98a84b } +.terminal-2456354039-r3 { fill: #9a9b99 } +.terminal-2456354039-r4 { fill: #608ab1 } +.terminal-2456354039-r5 { fill: #d0b344 } +.terminal-2456354039-r6 { fill: #98a84b;font-weight: bold } +.terminal-2456354039-r7 { fill: #868887;font-weight: bold } +.terminal-2456354039-r8 { fill: #868887 } +.terminal-2456354039-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/3_8 nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 93 params) -INFO    [✓] Input parameters look valid + + $ nf-core schema validate nf-core-rnaseq/3_8 nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index fd20914cfe..52d5e7c39f 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -19,89 +19,89 @@ font-weight: 700; } - .terminal-694979996-matrix { + .terminal-695111068-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-694979996-title { + .terminal-695111068-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-694979996-r1 { fill: #c5c8c6 } -.terminal-694979996-r2 { fill: #98a84b } -.terminal-694979996-r3 { fill: #9a9b99 } -.terminal-694979996-r4 { fill: #608ab1 } -.terminal-694979996-r5 { fill: #d0b344 } -.terminal-694979996-r6 { fill: #68a0b3;font-weight: bold } + .terminal-695111068-r1 { fill: #c5c8c6 } +.terminal-695111068-r2 { fill: #98a84b } +.terminal-695111068-r3 { fill: #9a9b99 } +.terminal-695111068-r4 { fill: #608ab1 } +.terminal-695111068-r5 { fill: #d0b344 } +.terminal-695111068-r6 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -113,30 +113,30 @@ - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Created component template: 'bam_stats_samtools' -INFO     Created following files:                                                                    -           subworkflows/nf-core/bam_stats_samtools/main.nf                                           -           subworkflows/nf-core/bam_stats_samtools/meta.yml                                          -           subworkflows/nf-core/bam_stats_samtools/tests/tags.yml                                    -           subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test                                + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Created component template: 'bam_stats_samtools' +INFO     Created following files:                                                                    +           subworkflows/nf-core/bam_stats_samtools/main.nf                                           +           subworkflows/nf-core/bam_stats_samtools/meta.yml                                          +           subworkflows/nf-core/bam_stats_samtools/tests/tags.yml                                    +           subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test                                diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg index b3f4f38ed3..e2c828102c 100644 --- a/docs/images/nf-core-subworkflows-info.svg +++ b/docs/images/nf-core-subworkflows-info.svg @@ -19,112 +19,112 @@ font-weight: 700; } - .terminal-1309193345-matrix { + .terminal-1309324417-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1309193345-title { + .terminal-1309324417-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1309193345-r1 { fill: #c5c8c6 } -.terminal-1309193345-r2 { fill: #98a84b } -.terminal-1309193345-r3 { fill: #9a9b99 } -.terminal-1309193345-r4 { fill: #608ab1 } -.terminal-1309193345-r5 { fill: #d0b344 } -.terminal-1309193345-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1309193345-r7 { fill: #98a84b;font-weight: bold } -.terminal-1309193345-r8 { fill: #868887 } -.terminal-1309193345-r9 { fill: #868887;font-style: italic; } -.terminal-1309193345-r10 { fill: #d08442 } -.terminal-1309193345-r11 { fill: #98729f } + .terminal-1309324417-r1 { fill: #c5c8c6 } +.terminal-1309324417-r2 { fill: #98a84b } +.terminal-1309324417-r3 { fill: #9a9b99 } +.terminal-1309324417-r4 { fill: #608ab1 } +.terminal-1309324417-r5 { fill: #d0b344 } +.terminal-1309324417-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1309324417-r7 { fill: #98a84b;font-weight: bold } +.terminal-1309324417-r8 { fill: #868887 } +.terminal-1309324417-r9 { fill: #868887;font-style: italic; } +.terminal-1309324417-r10 { fill: #d08442 } +.terminal-1309324417-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -136,36 +136,36 @@ - + - - $ nf-core subworkflows info bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -[..truncated..] - readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt -                                     │summary                            │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - versions  (file)                   │File containing software versions  │             versions.yml -                                     ╵                                   ╵                           - - 💻  Installation command: nf-core subworkflows install bam_rseqc - + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +[..truncated..] + readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt +                                     │summary                            │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + versions  (file)                   │File containing software versions  │             versions.yml +                                     ╵                                   ╵                           + + 💻  Installation command: nf-core subworkflows install bam_rseqc + diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg index 8c86c3a7e4..628ed9fef0 100644 --- a/docs/images/nf-core-subworkflows-install.svg +++ b/docs/images/nf-core-subworkflows-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + - - $ nf-core subworkflows install bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Installing 'bam_rseqc' -INFO     Use the following statement to include this subworkflow:                                    - - include { BAM_RSEQC } from '../subworkflows/nf-core/bam_rseqc/main'                                 - + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Installing 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-lint.svg b/docs/images/nf-core-subworkflows-lint.svg index 7e827661ac..a4497c6cd2 100644 --- a/docs/images/nf-core-subworkflows-lint.svg +++ b/docs/images/nf-core-subworkflows-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core subworkflows lint bam_stats_samtools - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting subworkflow: 'bam_stats_samtools' - -╭─[!] 14 Subworkflow Test Warnings───────────────────────────────────────────────────────────────╮ -│                     ╷                                     ╷                                      │ -│Subworkflow name   │File path                          │Test message                        │ -│╶────────────────────┼─────────────────────────────────────┼─────────────────────────────────────╴│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: //Add │ -│││all required assertions to verify │ -│││the test output.│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Add │ -│││tags for all modules used within │ -│││this subworkflow. Example:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Change │ -│││the test name preferably indicating │ -│││the test-data and file-format used│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Once │ -│││you have added the required tests, │ -│││please run the following command to │ -│││build this file:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: define │ -│││inputs of the workflow here. │ -│││Example:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: A │ -│││subworkflow SHOULD import at least │ -│││two modules│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: If in doubt │ -│││look at other nf-core/subworkflows │ -│││to see how we are doing things! :)│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit emitted│ -│││channels│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit input │ -│││(take) channels│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: substitute │ -│││modules here for the modules of your│ -│││subworkflow│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a │ -│││description of the subworkflow and │ -│││list keywords│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a list│ -│││of the modules and/or subworkflows │ -│││used in the subworkflow│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ -│││of the channels used as input with a│ -│││description and their structure│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ -│││of the channels used as output with │ -│││a descriptions and their structure│ -│                     ╵                                     ╵                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  42 Tests Passed │ -│[!]  14 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core subworkflows lint bam_stats_samtools + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting subworkflow: 'bam_stats_samtools' + +╭─[!] 15 Subworkflow Test Warnings───────────────────────────────────────────────────────────────╮ +│                     ╷                                     ╷                                      │ +│Subworkflow name   │File path                          │Test message                        │ +│╶────────────────────┼─────────────────────────────────────┼─────────────────────────────────────╴│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: //Add │ +│││all required assertions to verify │ +│││the test output.│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Add │ +│││tags for all modules used within │ +│││this subworkflow. Example:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Change │ +│││the test name preferably indicating │ +│││the test-data and file-format used│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Once │ +│││you have added the required tests, │ +│││please run the following command to │ +│││build this file:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: define │ +│││inputs of the workflow here. │ +│││Example:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: A │ +│││subworkflow SHOULD import at least │ +│││two modules│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: If in doubt │ +│││look at other nf-core/subworkflows │ +│││to see how we are doing things! :)│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit emitted│ +│││channels│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit input │ +│││(take) channels│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: substitute │ +│││modules here for the modules of your│ +│││subworkflow│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a │ +│││description of the subworkflow and │ +│││list keywords│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a list│ +│││of the modules and/or subworkflows │ +│││used in the subworkflow│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ +│││of the channels used as input with a│ +│││description and their structure│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ +│││of the channels used as output with │ +│││a descriptions and their structure│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│versions not found in snapshot file │ +│                     ╵                                     ╵                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  42 Tests Passed │ +│[!]  15 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg index 6bec883e1f..50072fb6df 100644 --- a/docs/images/nf-core-subworkflows-list-local.svg +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2930702375-matrix { + .terminal-1791436705-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2930702375-title { + .terminal-1791436705-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2930702375-r1 { fill: #c5c8c6 } -.terminal-2930702375-r2 { fill: #98a84b } -.terminal-2930702375-r3 { fill: #9a9b99 } -.terminal-2930702375-r4 { fill: #608ab1 } -.terminal-2930702375-r5 { fill: #d0b344 } -.terminal-2930702375-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2930702375-r7 { fill: #868887;font-style: italic; } + .terminal-1791436705-r1 { fill: #c5c8c6 } +.terminal-1791436705-r2 { fill: #98a84b } +.terminal-1791436705-r3 { fill: #9a9b99 } +.terminal-1791436705-r4 { fill: #608ab1 } +.terminal-1791436705-r5 { fill: #d0b344 } +.terminal-1791436705-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1791436705-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core subworkflows list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Subworkflows installed in '.':                                                              - -self.repo_type='pipeline' -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Subworkflow Name   ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ utils_nextflow_pip… │ https://github.co… │ 5caf7640a9ef1d18d7… │ remove             │ 2024-02-28 │ -│                     │                    │                     │ params.outdir from │            │ -│                     │                    │                     │ modules and        │            │ -│                     │                    │                     │ subworfklows tests │            │ -│                     │                    │                     │ (#5007)            │            │ -│ utils_nfcore_pipel… │ https://github.co… │ 5caf7640a9ef1d18d7… │ remove             │ 2024-02-28 │ -│                     │                    │                     │ params.outdir from │            │ -│                     │                    │                     │ modules and        │            │ -[..truncated..] + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: pipeline +INFO     Subworkflows installed in '.':                                                              + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Subworkflow Name        ┃Repository     ┃Version SHA┃Message                 ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ utils_nextflow_pipeline  │ nf-core/modules │ 5caf764     │ remove params.outdir     │ 2024-02-28 │ +│                          │                 │             │ from modules and         │            │ +│                          │                 │             │ subworfklows tests       │            │ +│                          │                 │             │ (#5007)                  │            │ +│ utils_nfcore_pipeline    │ nf-core/modules │ 92de218     │ handle multiple DOIs     │ 2024-05-02 │ +│                          │                 │             │ (#5556)                  │            │ +│ utils_nfvalidation_plug… │ nf-core/modules │ 5caf764     │ remove params.outdir     │ 2024-02-28 │ +│                          │                 │             │ from modules and         │            │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg index 704c2e9a45..b2c9fcb327 100644 --- a/docs/images/nf-core-subworkflows-list-remote.svg +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2294237964-matrix { + .terminal-1181242108-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2294237964-title { + .terminal-1181242108-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2294237964-r1 { fill: #c5c8c6 } -.terminal-2294237964-r2 { fill: #98a84b } -.terminal-2294237964-r3 { fill: #9a9b99 } -.terminal-2294237964-r4 { fill: #608ab1 } -.terminal-2294237964-r5 { fill: #d0b344 } -.terminal-2294237964-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-2294237964-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2294237964-r8 { fill: #868887;font-style: italic; } + .terminal-1181242108-r1 { fill: #c5c8c6 } +.terminal-1181242108-r2 { fill: #98a84b } +.terminal-1181242108-r3 { fill: #9a9b99 } +.terminal-1181242108-r4 { fill: #608ab1 } +.terminal-1181242108-r5 { fill: #d0b344 } +.terminal-1181242108-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-1181242108-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1181242108-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core subworkflows list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Subworkflow Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bam_cnv_wisecondorx                           │ -│ bam_create_som_pon_gatk                       │ -│ bam_dedup_stats_samtools_umitools             │ -│ bam_docounts_contamination_angsd              │ -│ bam_markduplicates_picard                     │ -│ bam_markduplicates_samtools                   │ -│ bam_ngscheckmate                              │ -│ bam_qc_picard                                 │ -│ bam_rseqc                                     │ -[..truncated..] + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_cnv_wisecondorx                           │ +│ bam_create_som_pon_gatk                       │ +│ bam_dedup_stats_samtools_umicollapse          │ +│ bam_dedup_stats_samtools_umitools             │ +│ bam_docounts_contamination_angsd              │ +│ bam_markduplicates_picard                     │ +│ bam_markduplicates_samtools                   │ +│ bam_ngscheckmate                              │ +│ bam_qc_picard                                 │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 42c576f4c5..0fa3b40f89 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     -INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     -INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         -INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           -'rseqc/junctionannotation'.                                                                 -INFO     Removed files for 'rseqc/junctionsaturation' and its dependencies                           -'rseqc/junctionsaturation'.                                                                 -INFO     Removed files for 'rseqc/readdistribution' and its dependencies 'rseqc/readdistribution'.   -INFO     Removed files for 'rseqc/readduplication' and its dependencies 'rseqc/readduplication'.     -INFO     Removed files for 'rseqc/tin' and its dependencies 'rseqc/tin'.                             -INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation,  -rseqc_junctionsaturation, rseqc_readdistribution, rseqc_readduplication, rseqc_tin'.        + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     +INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     +INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         +INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           +'rseqc/junctionannotation'.                                                                 +INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 9cda6beda6..a497ee900d 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-1767175273-matrix { + .terminal-1767306345-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1767175273-title { + .terminal-1767306345-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1767175273-r1 { fill: #c5c8c6 } -.terminal-1767175273-r2 { fill: #98a84b } -.terminal-1767175273-r3 { fill: #9a9b99 } -.terminal-1767175273-r4 { fill: #608ab1 } -.terminal-1767175273-r5 { fill: #d0b344 } + .terminal-1767306345-r1 { fill: #c5c8c6 } +.terminal-1767306345-r2 { fill: #98a84b } +.terminal-1767306345-r3 { fill: #9a9b99 } +.terminal-1767306345-r4 { fill: #608ab1 } +.terminal-1767306345-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Generating nf-test snapshot                                                                 + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Generating nf-test snapshot                                                                 diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index 3398899b7e..bff01d365b 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO    'subworkflows/nf-core/bam_rseqc' is already up to date                                      -INFO    'subworkflows/nf-core/utils_nextflow_pipeline' is already up to date                        -INFO    'subworkflows/nf-core/utils_nfcore_pipeline' is already up to date                          -INFO    'subworkflows/nf-core/utils_nfvalidation_plugin' is already up to date                      -INFO     Updates complete ✨                                                                         + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +ERROR   'rseqc/junctionsaturation' diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index b2c6d734fc..3a198f5713 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -WARNING  Could not find GitHub authentication token. Some API requests may fail.                     -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     -INFO     Committed changes to 'TEMPLATE' branch                                                      -INFO     Checking out original branch: 'master' -INFO     Now try to merge the updates in to your pipeline:                                           -           cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -           git merge TEMPLATE                                                                        + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +WARNING  Could not find GitHub authentication token. Some API requests may fail.                     +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' diff --git a/nf_core/__init__.py b/nf_core/__init__.py index d96be73f3d..2d4fe45a0a 100644 --- a/nf_core/__init__.py +++ b/nf_core/__init__.py @@ -1,4 +1,4 @@ -""" Main nf_core module file. +"""Main nf_core module file. Shouldn't do much, as everything is under subcommands. """ diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 7d2d083fa9..147b0586b2 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -""" nf-core: Helper tools for use with nf-core Nextflow pipelines. """ +"""nf-core: Helper tools for use with nf-core Nextflow pipelines.""" + import logging import os import sys @@ -102,6 +103,14 @@ def selective_traceback_hook(exctype, value, traceback): sys.excepthook = selective_traceback_hook +# Define callback function to normalize the case of click arguments, +# which is used to make the module/subworkflow names, provided by the +# user on the cli, case insensitive. +def normalize_case(ctx, param, component_name): + if component_name is not None: + return component_name.casefold() + + def run_nf_core(): # print nf-core header if environment variable is not set if os.environ.get("_NF_CORE_COMPLETE") is None: @@ -357,11 +366,19 @@ def create_params_file(pipeline, revision, output, force, show_hidden): help="Archive compression type", ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +# TODO: Remove this in a future release. Deprecated in March 2024. @click.option( "-t", "--tower", is_flag=True, default=False, + hidden=True, + help="Download for Seqera Platform. DEPRECATED: Please use `--platform` instead.", +) +@click.option( + "--platform", + is_flag=True, + default=False, help="Download for Seqera Platform (formerly Nextflow Tower)", ) @click.option( @@ -369,7 +386,12 @@ def create_params_file(pipeline, revision, output, force, show_hidden): "--download-configuration", is_flag=True, default=False, - help="Include configuration profiles in download. Not available with `--tower`", + help="Include configuration profiles in download. Not available with `--platform`", +) +@click.option( + "--tag", + multiple=True, + help="Add custom alias tags to `--platform` downloads. For example, `--tag \"3.10=validated\"` adds the custom 'validated' tag to the 3.10 release.", ) # -c changed to -s for consistency with other --container arguments, where it is always the first letter of the last word. # Also -c might be used instead of -d for config in a later release, but reusing params for different options in two subsequent releases might be too error-prone. @@ -411,7 +433,9 @@ def download( compress, force, tower, + platform, download_configuration, + tag, container_system, container_library, container_cache_utilisation, @@ -426,14 +450,18 @@ def download( """ from nf_core.download import DownloadWorkflow + if tower: + log.warning("[red]The `-t` / `--tower` flag is deprecated. Please use `--platform` instead.[/]") + dl = DownloadWorkflow( pipeline, revision, outdir, compress, force, - tower, + tower or platform, # True if either specified download_configuration, + tag, container_system, container_library, container_cache_utilisation, @@ -776,7 +804,7 @@ def modules_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-b # nf-core modules install @modules.command("install") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -828,7 +856,7 @@ def modules_install(ctx, tool, dir, prompt, force, sha): # nf-core modules update @modules.command("update") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -920,7 +948,7 @@ def modules_update( # nf-core modules patch @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -957,7 +985,7 @@ def patch(ctx, tool, dir, remove): # nf-core modules remove @modules.command("remove") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1111,7 +1139,7 @@ def create_module( # nf-core modules test @modules.command("test") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1170,7 +1198,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once, profile): # nf-core modules lint @modules.command("lint") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1257,7 +1285,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, # nf-core modules info @modules.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1296,7 +1324,7 @@ def modules_info(ctx, tool, dir): # nf-core modules bump-versions @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1382,7 +1410,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): # nf-core subworkflows test @subworkflows.command("test") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1509,7 +1537,7 @@ def subworkflows_list_local(ctx, keywords, json, dir): # pylint: disable=redefi # nf-core subworkflows lint @subworkflows.command("lint") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1590,7 +1618,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1598,7 +1626,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def subworkflows_info(ctx, tool, dir): +def subworkflows_info(ctx, subworkflow, dir): """ Show developer usage information about a given subworkflow. @@ -1615,7 +1643,7 @@ def subworkflows_info(ctx, tool, dir): try: subworkflow_info = SubworkflowInfo( dir, - tool, + subworkflow, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], @@ -1629,7 +1657,7 @@ def subworkflows_info(ctx, tool, dir): # nf-core subworkflows install @subworkflows.command("install") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1687,7 +1715,7 @@ def subworkflows_install(ctx, subworkflow, dir, prompt, force, sha): # nf-core subworkflows remove @subworkflows.command("remove") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1717,7 +1745,7 @@ def subworkflows_remove(ctx, dir, subworkflow): # nf-core subworkflows update @subworkflows.command("update") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -2119,10 +2147,16 @@ def logo(logo_text, dir, name, theme, width, format, force): default=False, help="Make a GitHub pull-request with the changes.", ) +@click.option( + "--force_pr", + is_flag=True, + default=False, + help="Force the creation of a pull-request, even if there are no changes.", +) @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") -def sync(dir, from_branch, pull_request, github_repository, username, template_yaml): +def sync(dir, from_branch, pull_request, github_repository, username, template_yaml, force_pr): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. @@ -2142,7 +2176,7 @@ def sync(dir, from_branch, pull_request, github_repository, username, template_y is_pipeline_directory(dir) # Sync the given pipeline dir - sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) + sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml, force_pr) try: sync_obj.sync() except (SyncExceptionError, PullRequestExceptionError) as e: diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 8332429835..4df67639e2 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -245,7 +245,7 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: # Update path in modules.json if the file is in the correct format modules_json = ModulesJson(self.dir) modules_json.load() - if modules_json.has_git_url_and_modules(): + if modules_json.has_git_url_and_modules() and modules_json.modules_json is not None: modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ self.modules_repo.repo_path ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index f1a9e7c401..9b81f54f06 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -2,7 +2,6 @@ The ComponentsTest class handles the generation and testing of nf-test snapshots. """ - import logging import os import re @@ -91,9 +90,9 @@ def run(self) -> None: """Run build steps""" self.check_inputs() os.environ["NFT_DIFF"] = "pdiff" # set nf-test differ to pdiff to get a better diff output - os.environ[ - "NFT_DIFF_ARGS" - ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + os.environ["NFT_DIFF_ARGS"] = ( + "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + ) with nf_core.utils.set_wd(Path(self.dir)): self.check_snapshot_stability() if len(self.errors) > 0: diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c4b477a0ab..6c9c01b496 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -2,7 +2,6 @@ The ComponentCreate class handles generating of module and subworkflow templates """ - import glob import json import logging @@ -395,7 +394,7 @@ def _get_component_dirs(self): if self.component_type == "modules": file_paths["environment.yml"] = component_dir / "environment.yml" file_paths["tests/tags.yml"] = component_dir / "tests" / "tags.yml" - file_paths["tests/main.nf.test"] = component_dir / "tests" / "main.nf.test" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 3c2fb9dde3..564dcfaf6b 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -3,7 +3,6 @@ in nf-core pipelines """ - import logging import operator import os @@ -12,6 +11,7 @@ import rich.box import rich.console import rich.panel +import rich.repr from rich.markdown import Markdown from rich.table import Table @@ -32,6 +32,7 @@ class LintExceptionError(Exception): pass +@rich.repr.auto class LintResult: """An object to hold the results of a lint test""" @@ -43,6 +44,7 @@ def __init__(self, component, lint_test, message, file_path): self.component_name = component.component_name +@rich.repr.auto class ComponentLint(ComponentCommand): """ An object for linting modules and subworkflows either in a clone of the 'nf-core/modules' @@ -232,9 +234,21 @@ def format_result(test_results, table): if last_modname and lint_result.component_name != last_modname: even_row = not even_row last_modname = lint_result.component_name + + # If this is an nf-core module, link to the nf-core webpage + if lint_result.component.repo_url == "https://github.com/nf-core/modules.git": + module_url = "https://nf-co.re/modules/" + lint_result.component_name.replace("/", "_") + module_name = f"[link={module_url}]{lint_result.component_name}[/link]" + else: + module_name = lint_result.component_name + + # Make the filename clickable to open in VSCode + file_path = os.path.relpath(lint_result.file_path, self.dir) + file_path_link = f"[link=vscode://file/{os.path.abspath(file_path)}]{file_path}[/link]" + table.add_row( - Markdown(f"{lint_result.component_name}"), - os.path.relpath(lint_result.file_path, self.dir), + module_name, + file_path_link, Markdown(f"{lint_result.message}"), style="dim" if even_row else None, ) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index b0c5af219f..f5f2744e17 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -70,7 +70,7 @@ def pattern_msg(keywords: List[str]) -> str: # We have a pipeline - list what's installed else: # Check that we are in a pipeline directory - print(f"{self.repo_type=}") + log.info(f"Repository type: [blue]{self.repo_type}") try: if self.repo_type != "pipeline": raise UserWarning( @@ -125,10 +125,11 @@ def pattern_msg(keywords: List[str]) -> str: version_sha = component_entry["git_sha"] try: # pass repo_name to get info on modules even outside nf-core/modules - message, date = ModulesRepo( + module = ModulesRepo( remote_url=repo_url, branch=component_entry["branch"], - ).get_commit_info(version_sha) + ) + message, date = module.get_commit_info(version_sha) except LookupError as e: log.warning(e) date = "[red]Not Available" @@ -140,7 +141,14 @@ def pattern_msg(keywords: List[str]) -> str: version_sha = "[red]Not Available" date = "[red]Not Available" message = "[red]Not Available" - table.add_row(component, repo_url, version_sha, message, date) + nice_repo_name = repo_url.replace("https://github.com/", "").replace(".git", "") + table.add_row( + component, + f"[link={repo_url}]{nice_repo_name}[/link]", + f"[link={module.gitless_repo()}/commit/{version_sha}]{version_sha[:7]}[/link]", + message, + date, + ) components.append(component) if print_json: diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 2f73afe9d3..bcba068afe 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -1,6 +1,7 @@ """ The NFCoreComponent class holds information and utility functions for a single module or subworkflow """ + import logging import re from pathlib import Path @@ -163,7 +164,7 @@ def get_inputs_from_main_nf(self): # path foo # don't match anything inside comments or after "output:" if "input:" not in data: - log.info(f"Could not find any inputs in {self.main_nf}") + log.debug(f"Could not find any inputs in {self.main_nf}") return inputs input_data = data.split("input:")[1].split("output:")[0] regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" @@ -175,7 +176,7 @@ def get_inputs_from_main_nf(self): elif match.group(4): input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases inputs.append(input_val) - log.info(f"Found {len(inputs)} inputs in {self.main_nf}") + log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") self.inputs = inputs def get_outputs_from_main_nf(self): @@ -184,12 +185,12 @@ def get_outputs_from_main_nf(self): data = f.read() # get output values from main.nf after "output:". the names are always after "emit:" if "output:" not in data: - log.info(f"Could not find any outputs in {self.main_nf}") + log.debug(f"Could not find any outputs in {self.main_nf}") return outputs output_data = data.split("output:")[1].split("when:")[0] regex = r"emit:\s*([^)\s,]+)" matches = re.finditer(regex, output_data, re.MULTILINE) for _, match in enumerate(matches, start=1): outputs.append(match.group(1)) - log.info(f"Found {len(outputs)} outputs in {self.main_nf}") + log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") self.outputs = outputs diff --git a/nf_core/create.py b/nf_core/create.py index c094d33a22..b420b1c86d 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -1,6 +1,7 @@ """Creates a nf-core pipeline matching the current organization's specification based on a template. """ + import configparser import logging import os diff --git a/nf_core/download.py b/nf_core/download.py index d08e0ba40e..f5ab3a0f58 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -88,10 +88,18 @@ class DownloadWorkflow: Args: pipeline (str): A nf-core pipeline name. - revision (List[str]): The workflow revision to download, like `1.0`. Defaults to None. - container (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. - tower (bool): Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + revision (List[str]): The workflow revision(s) to download, like `1.0` or `dev` . Defaults to None. outdir (str): Path to the local download directory. Defaults to None. + compress_type (str): Type of compression for the downloaded files. Defaults to None. + force (bool): Flag to force download even if files already exist (overwrite existing files). Defaults to False. + platform (bool): Flag to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. + download_configuration (str): Download the configuration files from nf-core/configs. Defaults to None. + tag (List[str]): Specify additional tags to add to the downloaded pipeline. Defaults to None. + container_system (str): The container system to use (e.g., "singularity"). Defaults to None. + container_library (List[str]): The container libraries (registries) to use. Defaults to None. + container_cache_utilisation (str): If a local or remote cache of already existing container images should be considered. Defaults to None. + container_cache_index (str): An index for the remote container cache. Defaults to None. + parallel_downloads (int): The number of parallel downloads to use. Defaults to 4. """ def __init__( @@ -101,8 +109,9 @@ def __init__( outdir=None, compress_type=None, force=False, - tower=False, + platform=False, download_configuration=None, + additional_tags=None, container_system=None, container_library=None, container_cache_utilisation=None, @@ -120,11 +129,20 @@ def __init__( self.output_filename = None self.compress_type = compress_type self.force = force - self.tower = tower + self.platform = platform # if flag is not specified, do not assume deliberate choice and prompt config inclusion interactively. - # this implies that non-interactive "no" choice is only possible implicitly (e.g. with --tower or if prompt is suppressed by !stderr.is_interactive). + # this implies that non-interactive "no" choice is only possible implicitly (e.g. with --platform or if prompt is suppressed by !stderr.is_interactive). # only alternative would have been to make it a parameter with argument, e.g. -d="yes" or -d="no". - self.include_configs = True if download_configuration else False if bool(tower) else None + self.include_configs = True if download_configuration else False if bool(platform) else None + # Additional tags to add to the downloaded pipeline. This enables to mark particular commits or revisions with + # additional tags, e.g. "stable", "testing", "validated", "production" etc. Since this requires a git-repo, it is only + # available for the bare / Seqera Platform download. + if isinstance(additional_tags, str) and bool(len(additional_tags)) and self.platform: + self.additional_tags = [additional_tags] + elif isinstance(additional_tags, tuple) and bool(len(additional_tags)) and self.platform: + self.additional_tags = [*additional_tags] + else: + self.additional_tags = None # Specifying a cache index or container library implies that containers should be downloaded. self.container_system = "singularity" if container_cache_index or bool(container_library) else container_system # Manually specified container library (registry) @@ -165,8 +183,8 @@ def download_workflow(self): ) self.prompt_revision() self.get_revision_hash() - # Inclusion of configs is unnecessary for Tower. - if not self.tower and self.include_configs is None: + # Inclusion of configs is unnecessary for Seqera Platform. + if not self.platform and self.include_configs is None: self.prompt_config_inclusion() # If a remote cache is specified, it is safe to assume images should be downloaded. if not self.container_cache_utilisation == "remote": @@ -177,7 +195,7 @@ def download_workflow(self): self.prompt_singularity_cachedir_utilization() self.prompt_singularity_cachedir_remote() # Nothing meaningful to compress here. - if not self.tower: + if not self.platform: self.prompt_compression_type() except AssertionError as e: raise DownloadError(e) from e @@ -196,7 +214,7 @@ def download_workflow(self): ) # Set an output filename now that we have the outdir - if self.tower: + if self.platform: self.output_filename = f"{self.outdir}.git" summary_log.append(f"Output file: '{self.output_filename}'") elif self.compress_type is not None: @@ -205,11 +223,11 @@ def download_workflow(self): else: summary_log.append(f"Output directory: '{self.outdir}'") - if not self.tower: + if not self.platform: # Only show entry, if option was prompted. summary_log.append(f"Include default institutional configuration: '{self.include_configs}'") else: - summary_log.append(f"Enabled for seqeralabs® Nextflow Tower: '{self.tower}'") + summary_log.append(f"Enabled for Seqera Platform: '{self.platform}'") # Check that the outdir doesn't already exist if os.path.exists(self.outdir): @@ -233,8 +251,8 @@ def download_workflow(self): log.info("Saving '{}'\n {}".format(self.pipeline, "\n ".join(summary_log))) # Perform the actual download - if self.tower: - self.download_workflow_tower() + if self.platform: + self.download_workflow_platform() else: self.download_workflow_static() @@ -273,7 +291,7 @@ def download_workflow_static(self): log.info("Compressing output into archive") self.compress_download() - def download_workflow_tower(self, location=None): + def download_workflow_platform(self, location=None): """Create a bare-cloned git repository of the workflow, so it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") @@ -282,6 +300,7 @@ def download_workflow_tower(self, location=None): remote_url=f"https://github.com/{self.pipeline}.git", revision=self.revision if self.revision else None, commit=self.wf_sha.values() if bool(self.wf_sha) else None, + additional_tags=self.additional_tags, location=(location if location else None), # manual location is required for the tests to work in_cache=False, ) @@ -289,7 +308,7 @@ def download_workflow_tower(self, location=None): # Remove tags for those revisions that had not been selected self.workflow_repo.tidy_tags_and_branches() - # create a bare clone of the modified repository needed for Tower + # create a bare clone of the modified repository needed for Seqera Platform self.workflow_repo.bare_clone(os.path.join(self.outdir, self.output_filename)) # extract the required containers @@ -306,9 +325,11 @@ def download_workflow_tower(self, location=None): except OSError as e: raise DownloadError(f"[red]{e}[/]") from e - # Justify why compression is skipped for Tower downloads (Prompt is not shown, but CLI argument could have been set) + # Justify why compression is skipped for Seqera Platform downloads (Prompt is not shown, but CLI argument could have been set) if self.compress_type is not None: - log.info("Compression choice is ignored for Tower downloads since nothing can be reasonably compressed.") + log.info( + "Compression choice is ignored for Seqera Platform downloads since nothing can be reasonably compressed." + ) def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" @@ -321,13 +342,13 @@ def prompt_revision(self): """ Prompt for pipeline revision / branch Prompt user for revision tag if '--revision' was not set - If --tower is specified, allow to select multiple revisions + If --platform is specified, allow to select multiple revisions Also the static download allows for multiple revisions, but we do not prompt this option interactively. """ if not bool(self.revision): (choice, tag_set) = nf_core.utils.prompt_pipeline_release_branch( - self.wf_revisions, self.wf_branches, multiple=self.tower + self.wf_revisions, self.wf_branches, multiple=self.platform ) """ The checkbox() prompt unfortunately does not support passing a Validator, @@ -384,7 +405,7 @@ def get_revision_hash(self): else: self.outdir = f"{self.pipeline.replace('/', '-').lower()}_{self.revision[0]}" - if not self.tower: + if not self.platform: for revision, wf_sha in self.wf_sha.items(): # Set the download URL and return - only applicable for classic downloads self.wf_download_url = { @@ -406,7 +427,7 @@ def prompt_config_inclusion(self): def prompt_container_download(self): """Prompt whether to download container images or not""" - if self.container_system is None and stderr.is_interactive and not self.tower: + if self.container_system is None and stderr.is_interactive and not self.platform: stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container_system = questionary.select( "Download software container images:", @@ -923,7 +944,7 @@ def rectify_raw_container_matches(self, raw_findings): if bool(container_definition) and bool(container_definition.group(1)): pattern = re.escape(container_definition.group(1)) # extract the quoted string(s) following the variable assignment - container_names = re.findall(r"%s\s*=\s*[\"\']([^\"\']+)[\"\']" % pattern, search_space) + container_names = re.findall(rf"{pattern}\s*=\s*[\"\']([^\"\']+)[\"\']", search_space) if bool(container_names): if isinstance(container_names, str): @@ -1487,6 +1508,7 @@ def __init__( remote_url, revision, commit, + additional_tags, location=None, hide_progress=False, in_cache=True, @@ -1521,13 +1543,21 @@ def __init__( self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache) - # expose some instance attributes - self.tags = self.repo.tags + # additional tags to be added to the repository + self.additional_tags = additional_tags if additional_tags else None def __repr__(self): """Called by print, creates representation of object""" return f"" + @property + def heads(self): + return self.repo.heads + + @property + def tags(self): + return self.repo.tags + def access(self): if os.path.exists(self.local_repo_dir): return self.local_repo_dir @@ -1621,7 +1651,7 @@ def setup_local_repo(self, remote, location=None, in_cache=True): def tidy_tags_and_branches(self): """ Function to delete all tags and branches that are not of interest to the downloader. - This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + This allows a clutter-free experience in Seqera Platform. The untagged commits are evidently still available. However, due to local caching, the downloader might also want access to revisions that had been deleted before. In that case, don't bother with re-adding the tags and rather download anew from Github. @@ -1638,7 +1668,6 @@ def tidy_tags_and_branches(self): # delete unwanted tags from repository for tag in tags_to_remove: self.repo.delete_tag(tag) - self.tags = self.repo.tags # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master") self.checkout(self.revision[0]) @@ -1655,7 +1684,7 @@ def tidy_tags_and_branches(self): if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) - # no branch exists, but one is required for Tower's UI to display revisions correctly). Thus, "latest" will be created. + # no branch exists, but one is required for Seqera Platform's UI to display revisions correctly). Thus, "latest" will be created. if not bool(self.repo.heads): if self.repo.is_valid_object("latest"): # "latest" exists as tag but not as branch @@ -1675,7 +1704,8 @@ def tidy_tags_and_branches(self): if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) - self.heads = self.repo.heads + # Apply the custom additional tags to the repository + self.__add_additional_tags() # get all tags and available remote_branches completed_revisions = {revision.name for revision in self.repo.heads + self.repo.tags} @@ -1693,6 +1723,39 @@ def tidy_tags_and_branches(self): self.retry_setup_local_repo(skip_confirm=True) raise DownloadError(e) from e + # "Private" method to add the additional custom tags to the repository. + def __add_additional_tags(self) -> None: + if self.additional_tags: + # example.com is reserved by the Internet Assigned Numbers Authority (IANA) as special-use domain names for documentation purposes. + # Although "dev-null" is a syntactically-valid local-part that is equally valid for delivery, + # and only the receiving MTA can decide whether to accept it, it is to my best knowledge configured with + # a Postfix discard mail delivery agent (https://www.postfix.org/discard.8.html), so incoming mails should be sinkholed. + self.ensure_git_user_config(f"nf-core download v{nf_core.__version__}", "dev-null@example.com") + + for additional_tag in self.additional_tags: + # A valid git branch or tag name can contain alphanumeric characters, underscores, hyphens, and dots. + # But it must not start with a dot, hyphen or underscore and also cannot contain two consecutive dots. + if re.match(r"^\w[\w_.-]+={1}\w[\w_.-]+$", additional_tag) and ".." not in additional_tag: + anchor, tag = additional_tag.split("=") + if self.repo.is_valid_object(anchor) and not self.repo.is_valid_object(tag): + try: + self.repo.create_tag( + tag, ref=anchor, message=f"Synonynmous tag to {anchor}; added by `nf-core download`." + ) + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n") + else: + if not self.repo.is_valid_object(anchor): + log.error( + f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit." + ) + if self.repo.is_valid_object(tag): + log.error( + f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{tag}' must not exist hitherto." + ) + else: + log.error(f"[red]Could not apply invalid `--tag` specification[/]: '{additional_tag}'") + def bare_clone(self, destination): if self.repo: try: diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index f04c7dcd04..6afca0e479 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,14 +1,14 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:c15ee2f4de8902421ccbfbd77329a9e8e4abf18477e1b5ae8a1a572790254647 +FROM gitpod/workspace-base@sha256:124f2b8cbefe9b4abbb6a14538da8846770dde20b93f038d9551b6230aec1d1c USER root # Install util tools. # software-properties-common is needed to add ppa support for Apptainer installation RUN apt-get update --quiet && \ - apt-get install --quiet --yes \ + apt-get install --quiet --yes --no-install-recommends \ apt-transport-https \ apt-utils \ sudo \ @@ -18,18 +18,17 @@ RUN apt-get update --quiet && \ curl \ tree \ graphviz \ - software-properties-common - -# Install Apptainer (Singularity) -RUN add-apt-repository -y ppa:apptainer/ppa && \ + software-properties-common && \ + add-apt-repository -y ppa:apptainer/ppa && \ apt-get update --quiet && \ - apt install -y apptainer - -# Install Conda -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + apt-get install --quiet --yes apptainer && \ + wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-latest-Linux-x86_64.sh + rm Miniconda3-latest-Linux-x86_64.sh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# Set PATH for Conda ENV PATH="/opt/conda/bin:$PATH" # Add the nf-core source files to the image @@ -47,23 +46,20 @@ RUN conda config --add channels defaults && \ conda config --add channels conda-forge && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base \ - mamba \ nextflow \ - nf-core \ nf-test \ prettier \ pre-commit \ ruff \ + mypy \ openjdk \ pytest-workflow && \ conda clean --all --force-pkgs-dirs --yes -# Update Nextflow -RUN nextflow self-update - -# Install nf-core -RUN python -m pip install . --no-cache-dir +# Update Nextflow and Install nf-core +RUN nextflow self-update && \ + python -m pip install . --no-cache-dir # Setup pdiff for nf-test diffs -RUN export NFT_DIFF="pdiff" && \ - export NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" +ENV NFT_DIFF="pdiff" +ENV NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" diff --git a/nf_core/launch.py b/nf_core/launch.py index 25bb4c150c..bc0cd58aec 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -1,5 +1,4 @@ -""" Launch a pipeline, interactively collecting params """ - +"""Launch a pipeline, interactively collecting params""" import copy import json diff --git a/nf_core/licences.py b/nf_core/licences.py index a8a35334dd..be737280f8 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -1,6 +1,5 @@ """Lists software licences for a given workflow.""" - import json import logging import os diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..9292a07fd8 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -9,7 +9,7 @@ import logging import os from pathlib import Path -from typing import List, Union +from typing import List, Tuple, Union import git import rich @@ -25,6 +25,7 @@ import nf_core.subworkflows.lint import nf_core.utils from nf_core import __version__ +from nf_core.components.lint import ComponentLint from nf_core.lint_utils import console from nf_core.utils import plural_s as _s from nf_core.utils import strip_ansi_codes @@ -32,148 +33,6 @@ log = logging.getLogger(__name__) -def run_linting( - pipeline_dir, - release_mode=False, - fix=(), - key=(), - show_passed=False, - fail_ignored=False, - fail_warned=False, - sort_by="test", - md_fn=None, - json_fn=None, - hide_progress=False, -): - """Runs all nf-core linting checks on a given Nextflow pipeline project - in either `release` mode or `normal` mode (default). Returns an object - of type :class:`PipelineLint` after finished. - - Args: - pipeline_dir (str): The path to the Nextflow pipeline root directory - release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. - See :class:`PipelineLint` for more information. - - Returns: - An object of type :class:`PipelineLint` that contains all the linting results. - An object of type :class:`ComponentLint` that contains all the linting results for the modules. - An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. - """ - - # Verify that the requested tests exist - if key: - all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( - set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) - ) - bad_keys = [k for k in key if k not in all_tests] - if len(bad_keys) > 0: - raise AssertionError( - "Test name{} not recognised: '{}'".format( - _s(bad_keys), - "', '".join(bad_keys), - ) - ) - log.info("Only running tests: '{}'".format("', '".join(key))) - - # Check if we were given any keys, and if they match any pipeline tests - if key: - pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) - else: - # If no key is supplied, run all tests - pipeline_keys = None - - # Create the lint object - lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) - - # Load the various pipeline configs - lint_obj._load_lint_config() - lint_obj._load_pipeline_config() - lint_obj._list_files() - - # Create the modules lint object - module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) - # Create the subworkflows lint object - try: - subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) - except LookupError: - subworkflow_lint_obj = None - - # Verify that the pipeline is correctly configured and has a modules.json file - module_lint_obj.has_valid_directory() - module_lint_obj.has_modules_file() - - # Run only the tests we want - if key: - # Select only the module lint tests - module_lint_tests = list( - set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) - ) - # Select only the subworkflow lint tests - subworkflow_lint_tests = list( - set(key).intersection( - set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) - ) - ) - else: - # If no key is supplied, run the default modules tests - module_lint_tests = ("module_changes", "module_version") - subworkflow_lint_tests = ("subworkflow_changes", "subworkflow_version") - module_lint_obj.filter_tests_by_key(module_lint_tests) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) - - # Set up files for component linting test - module_lint_obj.set_up_pipeline_files() - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.set_up_pipeline_files() - - # Run the pipeline linting tests - try: - lint_obj._lint_pipeline() - except AssertionError as e: - log.critical(f"Critical error: {e}") - log.info("Stopping tests...") - return lint_obj, module_lint_obj - - # Run the module lint tests - if len(module_lint_obj.all_local_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) - if len(module_lint_obj.all_remote_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) - # Run the subworkflows lint tests - if subworkflow_lint_obj is not None: - if len(subworkflow_lint_obj.all_local_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) - if len(subworkflow_lint_obj.all_remote_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) - - # Print the results - lint_obj._print_results(show_passed) - module_lint_obj._print_results(show_passed, sort_by=sort_by) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) - nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) - nf_core.lint_utils.print_fixes(lint_obj) - - # Save results to Markdown file - if md_fn is not None: - log.info(f"Writing lint results to {md_fn}") - markdown = lint_obj._get_results_md() - with open(md_fn, "w") as fh: - fh.write(markdown) - - # Save results to JSON file - if json_fn is not None: - lint_obj._save_json_results(json_fn) - - # Reminder about --release mode flag if we had failures - if len(lint_obj.failed) > 0: - if release_mode: - log.info("Reminder: Lint tests were run in --release mode.") - - return lint_obj, module_lint_obj, subworkflow_lint_obj - - class PipelineLint(nf_core.utils.Pipeline): """Object to hold linting information and results. @@ -199,6 +58,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .actions_schema_validation import ( # type: ignore[misc] actions_schema_validation, ) + from .configs import base_config, modules_config # type: ignore[misc] from .files_exist import files_exist # type: ignore[misc] from .files_unchanged import files_unchanged # type: ignore[misc] from .merge_markers import merge_markers # type: ignore[misc] @@ -206,6 +66,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] from .nextflow_config import nextflow_config # type: ignore[misc] + from .nfcore_yml import nfcore_yml # type: ignore[misc] from .pipeline_name_conventions import ( # type: ignore[misc] pipeline_name_conventions, ) @@ -264,6 +125,9 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "base_config", + "modules_config", + "nfcore_yml", ] + (["version_consistency"] if release_mode else []) def _load(self): @@ -409,7 +273,7 @@ def format_result(test_results): # Table of passed tests if len(self.passed) > 0 and show_passed: console.print( - rich.panel.Panel( + Panel( format_result(self.passed), title=rf"[bold][✔] {len(self.passed)} Pipeline Test{_s(self.passed)} Passed", title_align="left", @@ -421,7 +285,7 @@ def format_result(test_results): # Table of fixed tests if len(self.fixed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.fixed), title=rf"[bold][?] {len(self.fixed)} Pipeline Test{_s(self.fixed)} Fixed", title_align="left", @@ -433,7 +297,7 @@ def format_result(test_results): # Table of ignored tests if len(self.ignored) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.ignored), title=rf"[bold][?] {len(self.ignored)} Pipeline Test{_s(self.ignored)} Ignored", title_align="left", @@ -445,7 +309,7 @@ def format_result(test_results): # Table of warning tests if len(self.warned) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.warned), title=rf"[bold][!] {len(self.warned)} Pipeline Test Warning{_s(self.warned)}", title_align="left", @@ -457,7 +321,7 @@ def format_result(test_results): # Table of failing tests if len(self.failed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.failed), title=rf"[bold][✗] {len(self.failed)} Pipeline Test{_s(self.failed)} Failed", title_align="left", @@ -638,3 +502,144 @@ def _wrap_quotes(self, files: Union[List[str], List[Path], Path]) -> str: files = [files] bfiles = [f"`{str(f)}`" for f in files] return " or ".join(bfiles) + + +def run_linting( + pipeline_dir, + release_mode: bool = False, + fix=(), + key=(), + show_passed: bool = False, + fail_ignored: bool = False, + fail_warned: bool = False, + sort_by: str = "test", + md_fn=None, + json_fn=None, + hide_progress: bool = False, +) -> Tuple[PipelineLint, ComponentLint, Union[ComponentLint, None]]: + """Runs all nf-core linting checks on a given Nextflow pipeline project + in either `release` mode or `normal` mode (default). Returns an object + of type :class:`PipelineLint` after finished. + + Args: + pipeline_dir (str): The path to the Nextflow pipeline root directory + release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. + See :class:`PipelineLint` for more information. + + Returns: + An object of type :class:`PipelineLint` that contains all the linting results. + An object of type :class:`ComponentLint` that contains all the linting results for the modules. + An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. + """ + + # Verify that the requested tests exist + if key: + all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( + set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) + ) + bad_keys = [k for k in key if k not in all_tests] + if len(bad_keys) > 0: + raise AssertionError( + "Test name{} not recognised: '{}'".format( + _s(bad_keys), + "', '".join(bad_keys), + ) + ) + log.info("Only running tests: '{}'".format("', '".join(key))) + + # Check if we were given any keys, and if they match any pipeline tests + if key: + pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) + else: + # If no key is supplied, run all tests + pipeline_keys = None + + # Create the lint object + lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) + + # Load the various pipeline configs + lint_obj._load_lint_config() + lint_obj._load_pipeline_config() + lint_obj._list_files() + + # Create the modules lint object + module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) + # Create the subworkflows lint object + try: + subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) + except LookupError: + subworkflow_lint_obj = None + + # Verify that the pipeline is correctly configured and has a modules.json file + module_lint_obj.has_valid_directory() + module_lint_obj.has_modules_file() + # Run only the tests we want + if key: + # Select only the module lint tests + module_lint_tests = list( + set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) + ) + # Select only the subworkflow lint tests + subworkflow_lint_tests = list( + set(key).intersection( + set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) + ) + ) + else: + # If no key is supplied, run the default modules tests + module_lint_tests = list(("module_changes", "module_version")) + subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version")) + module_lint_obj.filter_tests_by_key(module_lint_tests) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) + + # Set up files for component linting test + module_lint_obj.set_up_pipeline_files() + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.set_up_pipeline_files() + + # Run the pipeline linting tests + try: + lint_obj._lint_pipeline() + except AssertionError as e: + log.critical(f"Critical error: {e}") + log.info("Stopping tests...") + return lint_obj, module_lint_obj, subworkflow_lint_obj + + # Run the module lint tests + if len(module_lint_obj.all_local_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) + if len(module_lint_obj.all_remote_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) + # Run the subworkflows lint tests + if subworkflow_lint_obj is not None: + if len(subworkflow_lint_obj.all_local_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) + if len(subworkflow_lint_obj.all_remote_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) + + # Print the results + lint_obj._print_results(show_passed) + module_lint_obj._print_results(show_passed, sort_by=sort_by) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) + nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) + nf_core.lint_utils.print_fixes(lint_obj) + + # Save results to Markdown file + if md_fn is not None: + log.info(f"Writing lint results to {md_fn}") + markdown = lint_obj._get_results_md() + with open(md_fn, "w") as fh: + fh.write(markdown) + + # Save results to JSON file + if json_fn is not None: + lint_obj._save_json_results(json_fn) + + # Reminder about --release mode flag if we had failures + if len(lint_obj.failed) > 0: + if release_mode: + log.info("Reminder: Lint tests were run in --release mode.") + + return lint_obj, module_lint_obj, subworkflow_lint_obj diff --git a/nf_core/lint/actions_awsfulltest.py b/nf_core/lint/actions_awsfulltest.py index 66aa3f99bf..d5a061c935 100644 --- a/nf_core/lint/actions_awsfulltest.py +++ b/nf_core/lint/actions_awsfulltest.py @@ -17,7 +17,7 @@ def actions_awsfulltest(self): .. note:: You can manually trigger the AWS tests by going to the `Actions` tab on the pipeline GitHub repository and selecting the `nf-core AWS full size tests` workflow on the left. - .. tip:: For tests on full data prior to release, `Nextflow Tower `_ launch feature can be employed. + .. tip:: For tests on full data prior to release, `Seqera Platform `_ launch feature can be employed. The ``.github/workflows/awsfulltest.yml`` file is tested for the following: @@ -52,7 +52,7 @@ def actions_awsfulltest(self): # Warn if `-profile test` is still unchanged try: - steps = wf["jobs"]["run-tower"]["steps"] + steps = wf["jobs"]["run-platform"]["steps"] if not any(aws_profile in step["run"] for step in steps if "run" in step.keys()): raise AssertionError() except (AssertionError, KeyError, TypeError): diff --git a/nf_core/lint/actions_schema_validation.py b/nf_core/lint/actions_schema_validation.py index fa4471d98c..7e878a1af8 100644 --- a/nf_core/lint/actions_schema_validation.py +++ b/nf_core/lint/actions_schema_validation.py @@ -1,13 +1,14 @@ import glob import logging import os +from typing import Any, Dict, List import jsonschema import requests import yaml -def actions_schema_validation(self): +def actions_schema_validation(self) -> Dict[str, List[str]]: """Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. @@ -17,8 +18,9 @@ def actions_schema_validation(self): To pass this test, make sure that all your workflows contain the required properties ``on`` and ``jobs`` and that all other properties are of the correct type, as specified in the schema (link above). """ - passed = [] - failed = [] + passed: List[str] = [] + failed: List[str] = [] + warned: List[str] = [] # Only show error messages from schema logging.getLogger("nf_core.schema").setLevel(logging.ERROR) @@ -28,7 +30,13 @@ def actions_schema_validation(self): # Load the GitHub workflow schema r = requests.get("https://json.schemastore.org/github-workflow", allow_redirects=True) - schema = r.json() + # handle "Service Unavailable" error + if r.status_code not in [200, 301]: + warned.append( + f"Failed to fetch schema: Response code for `https://json.schemastore.org/github-workflow` was {r.status_code}" + ) + return {"passed": passed, "failed": failed, "warned": warned} + schema: Dict[str, Any] = r.json() # Validate all workflows against the schema for wf_path in action_workflows: @@ -46,7 +54,7 @@ def actions_schema_validation(self): try: wf_json["on"] = wf_json.pop(True) except Exception: - failed.append("Missing 'on' keyword in {}.format(wf)") + failed.append(f"Missing 'on' keyword in {wf}") # Validate the workflow try: @@ -55,4 +63,4 @@ def actions_schema_validation(self): except Exception as e: failed.append(f"Workflow validation failed for {wf}: {e}") - return {"passed": passed, "failed": failed} + return {"passed": passed, "failed": failed, "warned": warned} diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py new file mode 100644 index 0000000000..2741529198 --- /dev/null +++ b/nf_core/lint/configs.py @@ -0,0 +1,101 @@ +import logging +import re +from pathlib import Path +from typing import Dict, List + +from nf_core.lint_utils import ignore_file + +log = logging.getLogger(__name__) + + +class LintConfig: + def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): + self.wf_path = wf_path + self.lint_config = lint_config + + def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: + """Lint a file and add the result to the passed or failed list.""" + + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + ignore_configs: List[str] = [] + + fn = Path(self.wf_path, file_path) + + passed, failed, ignored, ignore_configs = ignore_file(lint_name, file_path, Path(self.wf_path)) + + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # find sections with a withName: prefix + sections = re.findall(r"withName:\s*['\"]?(\w+)['\"]?", config) + log.debug(f"found sections: {sections}") + + # find all .nf files in the workflow directory + nf_files = list(Path(self.wf_path).rglob("*.nf")) + log.debug(f"found nf_files: {[str(f) for f in nf_files]}") + + # check if withName sections are present in config, but not in workflow files + for section in sections: + if section not in ignore_configs or section.lower() not in ignore_configs: + if not any(section in nf_file.read_text() for nf_file in nf_files): + failed.append( + f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the Nextflow scripts." + ) + else: + passed.append(f"`{section}` found in `{file_path}` and Nextflow scripts.") + else: + ignored.append(f"``{section}` is ignored.") + + return {"passed": passed, "failed": failed, "ignored": ignored} + + +def modules_config(self) -> Dict[str, List[str]]: + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + modules_config: False + + To disable this test only for specific modules, you can specify a list of module names. + + .. code-block:: yaml + + lint: + modules_config: + - fastqc + + """ + + result = LintConfig(self.wf_path, self.lint_config).lint_file("modules_config", Path("conf", "modules.config")) + + return result + + +def base_config(self) -> Dict[str, List[str]]: + """Make sure the conf/base.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + base_config: False + + """ + + result = LintConfig(self.wf_path, self.lint_config).lint_file("base_config", Path("conf", "base.config")) + + return result diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 5d62a23bf8..d801caf704 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -66,31 +66,30 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: conf/igenomes.config .github/workflows/awstest.yml .github/workflows/awsfulltest.yml - pyproject.toml Files that *must not* be present, due to being renamed or removed in the template: .. code-block:: bash - Singularity - parameters.settings.json - pipeline_template.yml # saving information in .nf-core.yml - .nf-core.yaml # NB: Should be yml, not yaml - bin/markdown_to_html.r - conf/aws.config - .github/workflows/push_dockerhub.yml .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/feature_request.md - docs/images/nf-core-PIPELINE_logo.png + .github/workflows/push_dockerhub.yml .markdownlint.yml + .nf-core.yaml # NB: Should be yml, not yaml .yamllint.yml + bin/markdown_to_html.r + conf/aws.config + docs/images/nf-core-PIPELINE_logo.png lib/Checks.groovy lib/Completion.groovy - lib/Workflow.groovy - lib/WorkflowPIPELINE.groovy lib/NfcoreTemplate.groovy lib/Utils.groovy + lib/Workflow.groovy lib/WorkflowMain.groovy + lib/WorkflowPIPELINE.groovy + parameters.settings.json + pipeline_template.yml # saving information in .nf-core.yml + Singularity Files that *should not* be present: @@ -176,30 +175,29 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], [Path("modules.json")], - [Path("pyproject.toml")], ] # List of strings. Fails / warns if any of the strings exist. files_fail_ifexists = [ - Path("Singularity"), - Path("parameters.settings.json"), - Path("pipeline_template.yml"), # saving information in .nf-core.yml - Path(".nf-core.yaml"), # yml not yaml - Path("bin", "markdown_to_html.r"), - Path("conf", "aws.config"), - Path(".github", "workflows", "push_dockerhub.yml"), Path(".github", "ISSUE_TEMPLATE", "bug_report.md"), Path(".github", "ISSUE_TEMPLATE", "feature_request.md"), - Path("docs", "images", f"nf-core-{short_name}_logo.png"), + Path(".github", "workflows", "push_dockerhub.yml"), Path(".markdownlint.yml"), + Path(".nf-core.yaml"), # yml not yaml Path(".yamllint.yml"), + Path("bin", "markdown_to_html.r"), + Path("conf", "aws.config"), + Path("docs", "images", f"nf-core-{short_name}_logo.png"), Path("lib", "Checks.groovy"), Path("lib", "Completion.groovy"), - Path("lib", "Workflow.groovy"), + Path("lib", "NfcoreTemplate.groovy"), Path("lib", "Utils.groovy"), + Path("lib", "Workflow.groovy"), Path("lib", "WorkflowMain.groovy"), - Path("lib", "NfcoreTemplate.groovy"), Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), + Path("parameters.settings.json"), + Path("pipeline_template.yml"), # saving information in .nf-core.yml + Path("Singularity"), ] files_warn_ifexists = [Path(".travis.yml")] files_fail_ifinconfig: List[Tuple[Path, Dict[str, str]]] = [ diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 3a3a0cb74a..014c2b5f09 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -47,7 +47,6 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: .gitignore .prettierignore - pyproject.toml .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting @@ -106,7 +105,7 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: [Path("docs", "README.md")], ] files_partial = [ - [Path(".gitignore"), Path(".prettierignore"), Path("pyproject.toml")], + [Path(".gitignore"), Path(".prettierignore")], ] # Only show error messages from pipeline creation @@ -161,10 +160,8 @@ def _tf(file_path: Union[str, Path]) -> Path: if filecmp.cmp(_pf(f), _tf(f), shallow=True): passed.append(f"`{f}` matches the template") else: - if ( - f.name.endswith(".png") - and os.stat(_pf(f)).st_mode == os.stat(_tf(f)).st_mode - and int(os.stat(_pf(f)).st_size / 100) == int(os.stat(_tf(f)).st_size / 100) + if f.name.endswith(".png") and int(os.stat(_pf(f)).st_size / 500) == int( + os.stat(_tf(f)).st_size / 500 ): # almost the same file, good enough for the logo log.debug(f"Files are almost the same. Will pass: {f}") diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 9b9c80c44e..8b4fa2120f 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -1,8 +1,10 @@ -import os +from pathlib import Path from typing import Dict, List import yaml +from nf_core.lint_utils import ignore_file + def multiqc_config(self) -> Dict[str, List[str]]: """Make sure basic multiQC plugins are installed and plots are exported @@ -21,100 +23,108 @@ def multiqc_config(self) -> Dict[str, List[str]]: order: -1001 export_plots: true - """ - - passed: List[str] = [] - failed: List[str] = [] - - # Remove field that should be ignored according to the linting config - ignore_configs = self.lint_config.get("multiqc_config", []) + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: - fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml") + .. code-block:: yaml - # Return a failed status if we can't find the file - if not os.path.isfile(fn): - return {"ignored": ["'assets/multiqc_config.yml' not found"]} + lint: + multiqc_config: False - try: - with open(fn) as fh: - mqc_yml = yaml.safe_load(fh) - except Exception as e: - return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} - - # check if requried sections are present - required_sections = ["report_section_order", "export_plots", "report_comment"] - for section in required_sections: - if section not in mqc_yml and section not in ignore_configs: - failed.append(f"'assets/multiqc_config.yml' does not contain `{section}`") - return {"passed": passed, "failed": failed} - else: - passed.append(f"'assets/multiqc_config.yml' contains `{section}`") - - try: - orders = {} - summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" - min_plugins = ["software_versions", summary_plugin_name] - for plugin in min_plugins: - if plugin not in mqc_yml["report_section_order"]: - raise AssertionError(f"Section {plugin} missing in report_section_order") - if "order" not in mqc_yml["report_section_order"][plugin]: - raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") - plugin_order = mqc_yml["report_section_order"][plugin]["order"] - if plugin_order >= 0: - raise AssertionError(f"Section {plugin} 'order' must be < 0") - - for plugin in mqc_yml["report_section_order"]: - if "order" in mqc_yml["report_section_order"][plugin]: - orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] - - if orders[summary_plugin_name] != min(orders.values()): - raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") - orders.pop(summary_plugin_name) - if orders["software_versions"] != min(orders.values()): - raise AssertionError("Section software_versions should have the second lowest order") - except (AssertionError, KeyError, TypeError) as e: - failed.append(f"'assets/multiqc_config.yml' does not meet requirements: {e}") - else: - passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.") - - if "report_comment" not in ignore_configs: - # Check that the minimum plugins exist and are coming first in the summary - version = self.nf_config.get("manifest.version", "").strip(" '\"") - if "dev" in version: - version = "dev" - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) + """ + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + fn = Path(self.wf_path, "assets", "multiqc_config.yml") + file_path = fn.relative_to(self.wf_path) + passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path) + + # skip other tests if the file is not found + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + mqc_yml = yaml.safe_load(fh) + except Exception as e: + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} + + # check if required sections are present + required_sections = ["report_section_order", "export_plots", "report_comment"] + for section in required_sections: + if section not in mqc_yml and section not in ignore_configs: + failed.append(f"`assets/multiqc_config.yml` does not contain `{section}`") + return {"passed": passed, "failed": failed} + else: + passed.append(f"`assets/multiqc_config.yml` contains `{section}`") + + try: + orders = {} + summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" + min_plugins = ["software_versions", summary_plugin_name] + for plugin in min_plugins: + if plugin not in mqc_yml["report_section_order"]: + raise AssertionError(f"Section {plugin} missing in report_section_order") + if "order" not in mqc_yml["report_section_order"][plugin]: + raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") + plugin_order = mqc_yml["report_section_order"][plugin]["order"] + if plugin_order >= 0: + raise AssertionError(f"Section {plugin} 'order' must be < 0") + + for plugin in mqc_yml["report_section_order"]: + if "order" in mqc_yml["report_section_order"][plugin]: + orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] + + if orders[summary_plugin_name] != min(orders.values()): + raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") + orders.pop(summary_plugin_name) + if orders["software_versions"] != min(orders.values()): + raise AssertionError("Section software_versions should have the second lowest order") + except (AssertionError, KeyError, TypeError) as e: + failed.append(f"`assets/multiqc_config.yml` does not meet requirements: {e}") else: - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) - - if mqc_yml["report_comment"].strip() != report_comments: - # find where the report_comment is wrong and give it as a hint - hint = report_comments - failed.append( - f"'assets/multiqc_config.yml' does not contain a matching 'report_comment'. \n" - f"The expected comment is: \n" - f"```{hint}``` \n" - f"The current comment is: \n" - f"```{ mqc_yml['report_comment'].strip()}```" - ) + passed.append("`assets/multiqc_config.yml` follows the ordering scheme of the minimally required plugins.") + + if "report_comment" not in ignore_configs: + # Check that the minimum plugins exist and are coming first in the summary + version = self.nf_config.get("manifest.version", "").strip(" '\"") + if "dev" in version: + version = "dev" + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + else: + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + if mqc_yml["report_comment"].strip() != report_comments: + # find where the report_comment is wrong and give it as a hint + hint = report_comments + failed.append( + f"`assets/multiqc_config.yml` does not contain a matching 'report_comment'. \n" + f"The expected comment is: \n" + f"```{hint}``` \n" + f"The current comment is: \n" + f"```{ mqc_yml['report_comment'].strip()}```" + ) + else: + passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.") + + # Check that export_plots is activated + try: + if not mqc_yml["export_plots"]: + raise AssertionError() + except (AssertionError, KeyError, TypeError): + failed.append("`assets/multiqc_config.yml` does not contain 'export_plots: true'.") else: - passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") - - # Check that export_plots is activated - try: - if not mqc_yml["export_plots"]: - raise AssertionError() - except (AssertionError, KeyError, TypeError): - failed.append("'assets/multiqc_config.yml' does not contain 'export_plots: true'.") - else: - passed.append("'assets/multiqc_config.yml' contains 'export_plots: true'.") - - return {"passed": passed, "failed": failed} + passed.append("`assets/multiqc_config.yml` contains 'export_plots: true'.") + + return {"passed": passed, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index d3e29d2363..47b7d78f5e 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -245,10 +245,9 @@ def nextflow_config(self): raise AssertionError() except (AssertionError, IndexError): failed.append( - "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( - manifest_homepage - ) + f"Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {manifest_homepage}" ) + else: passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") @@ -375,7 +374,6 @@ def nextflow_config(self): schema.load_schema() schema.get_schema_defaults() # Get default values from schema schema.get_schema_types() # Get types from schema - self.nf_config.keys() # Params in nextflow.config for param_name in schema.schema_defaults.keys(): param = "params." + param_name if param in ignore_defaults: diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py new file mode 100644 index 0000000000..f23b2f1a84 --- /dev/null +++ b/nf_core/lint/nfcore_yml.py @@ -0,0 +1,75 @@ +import re +from pathlib import Path +from typing import Dict, List + +from nf_core import __version__ + +REPOSITORY_TYPES = ["pipeline", "modules"] + + +def nfcore_yml(self) -> Dict[str, List[str]]: + """Repository ``.nf-core.yml`` tests + + The ``.nf-core.yml`` contains metadata for nf-core tools to correctly apply its features. + + * repository type: + + * Check that the repository type is set. + + * nf core version: + + * Check if the nf-core version is set to the latest version. + + """ + passed: List[str] = [] + warned: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get(".nf-core", []) + + try: + with open(Path(self.wf_path, ".nf-core.yml")) as fh: + content = fh.read() + except FileNotFoundError: + with open(Path(self.wf_path, ".nf-core.yaml")) as fh: + content = fh.read() + + if "repository_type" not in ignore_configs: + # Check that the repository type is set in the .nf-core.yml + repo_type_re = r"repository_type: (.+)" + match = re.search(repo_type_re, content) + if match: + repo_type = match.group(1) + if repo_type not in REPOSITORY_TYPES: + failed.append( + f"Repository type in `.nf-core.yml` is not valid. " + f"Should be one of `[{', '.join(REPOSITORY_TYPES)}]` but was `{repo_type}`" + ) + else: + passed.append(f"Repository type in `.nf-core.yml` is valid: `{repo_type}`") + else: + warned.append("Repository type not set in `.nf-core.yml`") + else: + ignored.append("`.nf-core.yml` variable ignored 'repository_type'") + + if "nf_core_version" not in ignore_configs: + # Check that the nf-core version is set in the .nf-core.yml + nf_core_version_re = r"nf_core_version: (.+)" + match = re.search(nf_core_version_re, content) + if match: + nf_core_version = match.group(1).strip('"') + if nf_core_version != __version__ and "dev" not in nf_core_version: + warned.append( + f"nf-core version in `.nf-core.yml` is not set to the latest version. " + f"Should be `{__version__}` but was `{nf_core_version}`" + ) + else: + passed.append(f"nf-core version in `.nf-core.yml` is set to the latest version: `{nf_core_version}`") + else: + warned.append("nf-core version not set in `.nf-core.yml`") + else: + ignored.append("`.nf-core.yml` variable ignored 'nf_core_version'") + + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 6eca6522d4..167600bfc0 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,6 +2,7 @@ import logging import subprocess from pathlib import Path +from typing import List import rich from rich.console import Console @@ -101,3 +102,30 @@ def dump_json_with_prettier(file_name, file_content): with open(file_name, "w") as fh: json.dump(file_content, fh, indent=4) run_prettier_on_file(file_name) + + +def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[str]]: + """Ignore a file and add the result to the ignored list. Return the passed, failed, ignored and ignore_configs lists.""" + + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + _, lint_conf = nf_core.utils.load_tools_config(dir_path) + lint_conf = lint_conf.get("lint", {}) + ignore_entry: List[str] | bool = lint_conf.get(lint_name, []) + full_path = dir_path / file_path + # Return a failed status if we can't find the file + if not full_path.is_file(): + if isinstance(ignore_entry, bool) and not ignore_entry: + ignored.append(f"`{file_path}` not found, but it is ignored.") + ignore_entry = [] + else: + failed.append(f"`{file_path}` not found.") + else: + passed.append(f"`{file_path}` found and not ignored.") + + # we handled the only case where ignore_entry should be a bool, convert it to a list, to make downstream code easier + if isinstance(ignore_entry, bool): + ignore_entry = [] + + return [passed, failed, ignored, ignore_entry] diff --git a/nf_core/list.py b/nf_core/list.py index 67d1a76878..658f4dc6d2 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -1,6 +1,5 @@ """Lists available nf-core pipelines and versions.""" - import json import logging import os diff --git a/nf_core/module-template/tests/main.nf.test b/nf_core/module-template/tests/main.nf.test.j2 similarity index 99% rename from nf_core/module-template/tests/main.nf.test rename to nf_core/module-template/tests/main.nf.test.j2 index e1b1dadf12..f31e92d659 100644 --- a/nf_core/module-template/tests/main.nf.test +++ b/nf_core/module-template/tests/main.nf.test.j2 @@ -29,7 +29,7 @@ nextflow_process { input[0] = [ [ id:'test', single_end:false ], // meta map file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] + ] {%- else %} input[0] = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) {%- endif %} diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index b9003be974..9b54174d5a 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -3,7 +3,6 @@ or for a single module """ - import logging import os import re diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 866e6312aa..f96683089c 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -6,7 +6,6 @@ nf-core modules lint """ - import logging import os @@ -214,6 +213,11 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals # Otherwise run all the lint tests else: + if self.repo_type == "pipeline" and self.modules_json: + # Set correct sha + version = self.modules_json.get_module_version(mod.component_name, mod.repo_url, mod.org) + mod.git_sha = version + for test_name in self.lint_tests: if test_name == "main_nf": getattr(self, test_name)(mod, fix_version, self.registry, progress_bar) diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index ee8cabebe1..eb76f4b88b 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -1,6 +1,7 @@ """ Check whether the content of a module has changed compared to the original repository """ + import shutil import tempfile from pathlib import Path diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 520f8cf0a2..b2b6c2221f 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -1,6 +1,7 @@ """ Lint the tests of a module in nf-core/modules """ + import json import logging from pathlib import Path @@ -106,6 +107,22 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + module.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + module.failed.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: module.failed.append( ( diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index f68c27b2d8..7d78268e92 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -6,7 +6,6 @@ import shutil import tempfile from pathlib import Path -from typing import Union import git import questionary @@ -32,7 +31,7 @@ class ModulesJson: An object for handling a 'modules.json' file in a pipeline """ - def __init__(self, pipeline_dir): + def __init__(self, pipeline_dir: str): """ Initialise the object. @@ -43,7 +42,7 @@ def __init__(self, pipeline_dir): self.modules_dir = Path(self.dir, "modules") self.subworkflows_dir = Path(self.dir, "subworkflows") self.modules_json_path = Path(self.dir, "modules.json") - self.modules_json: Union(dict, None) = None + self.modules_json = None self.pipeline_modules = None self.pipeline_subworkflows = None self.pipeline_components = None @@ -1051,17 +1050,18 @@ def get_component_branch(self, component_type, component, repo_url, install_dir) ) return branch - def dump(self, run_prettier: bool = False): + def dump(self, run_prettier: bool = False) -> None: """ Sort the modules.json, and write it to file """ - # Sort the modules.json - self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - if run_prettier: - dump_json_with_prettier(self.modules_json_path, self.modules_json) - else: - with open(self.modules_json_path, "w") as fh: - json.dump(self.modules_json, fh, indent=4) + if self.modules_json is not None: + # Sort the modules.json + self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) + if run_prettier: + dump_json_with_prettier(self.modules_json_path, self.modules_json) + else: + with open(self.modules_json_path, "w") as fh: + json.dump(self.modules_json, fh, indent=4) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 204c20fd71..9694920274 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -69,6 +69,12 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.avail_module_names = None + def gitless_repo(self): + gitless_repo_url = self.remote_url + if self.remote_url and ".git" in self.remote_url: + gitless_repo_url = gitless_repo_url[:-4] + return gitless_repo_url + def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): """ Sets up the local git repository. If the repository has been cloned previously, it diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index ca8993483b..6796de41ec 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -20,20 +20,19 @@ def repo_full_name_from_remote(remote_url: str) -> str: Extracts the path from the remote URL See https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS for the possible URL patterns """ - # Check whether we have a https or ssh url - if remote_url.startswith("https"): - path = urlparse(remote_url).path - # Remove the intial '/' - path = path[1:] - # Remove extension - path = os.path.splitext(path)[0] + + if remote_url.startswith(("https://", "http://", "ftps://", "ftp://", "ssh://")): + # Parse URL and remove the initial '/' + path = urlparse(remote_url).path.lstrip("/") + elif "git@" in remote_url: + # Extract the part after 'git@' and parse it + path = urlparse(remote_url.split("git@")[-1]).path else: - # Remove the initial `git@`` - split_path: list = remote_url.split("@") - path = split_path[-1] if len(split_path) > 1 else split_path[0] - path = urlparse(path).path - # Remove extension - path = os.path.splitext(path)[0] + path = urlparse(remote_url).path + + # Remove the file extension from the path + path, _ = os.path.splitext(path) + return path diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 267fe7086a..78798b065e 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -1,5 +1,4 @@ -""" Create a YAML parameter file """ - +"""Create a YAML parameter file""" import json import logging diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index dd9ffa5387..72dda289a0 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -28,10 +28,6 @@ indent_style = unset [/assets/email*] indent_size = unset -# ignore Readme -[README.md] -indent_style = unset - -# ignore python +# ignore python and markdown [*.{py,md}] indent_style = unset diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index d04335275a..4f01a97993 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 4c9fd69fcc..56ecb60302 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -8,12 +8,12 @@ on: types: [published] workflow_dispatch: jobs: - run-tower: + run-platform: name: Run AWS full tests if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - - name: Launch workflow via tower + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) @@ -33,7 +33,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: Tower debug log file + name: Seqera Platform debug log file path: | - tower_action_*.log - tower_action_*.json + seqera_platform_action_*.log + seqera_platform_action_*.json diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 25726aa1c9..e1c26a71c7 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -5,13 +5,13 @@ name: nf-core AWS test on: workflow_dispatch: jobs: - run-tower: + run-platform: name: Run AWS tests if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - # Launch workflow using Tower CLI tool action {%- raw %} - - name: Launch workflow via tower + # Launch workflow using Seqera Platform CLI tool action {%- raw %} + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} @@ -27,7 +27,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: Tower debug log file + name: Seqera Platform debug log file path: | - tower_action_*.log - tower_action_*.json + seqera_platform_action_*.log + seqera_platform_action_*.json diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 84c727f60d..6b2547765d 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,10 +28,10 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index dcd7caabfc..ebea16c5cb 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -14,6 +14,8 @@ on: pull_request: types: - opened + - edited + - synchronize branches: - master pull_request_target: @@ -28,11 +30,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Disk space cleanup + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: @@ -65,8 +70,17 @@ jobs: - name: Inspect download run: tree ./${{ env.REPOTITLE_LOWERCASE }} - - name: Run the downloaded pipeline + - name: Run the downloaded pipeline (stub) + id: stub_run_pipeline + continue-on-error: true + env: + NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results + - name: Run the downloaded pipeline (stub run not supported) + id: run_pipeline + if: ${{ job.steps.stub_run_pipeline.status == failure() }} env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results{% endraw %} + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results{% endraw %} diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 28e6605b96..18e6f9e158 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -32,9 +32,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 59b85f95fc..d1ecae12b7 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,12 +14,12 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit @@ -32,14 +32,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - name: Install dependencies @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: linting-logs path: | diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index bb3eafcc2c..ea408fd6f8 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 + uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3 with: workflow: linting.yml workflow_conclusion: completed diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index 58ef1731f7..8fee061fdd 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -12,7 +12,7 @@ jobs: - name: get topics and convert to hashtags id: get_topics run: | - curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' >> $GITHUB_OUTPUT + echo "topics=$(curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ')" >> $GITHUB_OUTPUT - uses: rzr/fediverse-action@master with: @@ -25,13 +25,13 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} - ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience #nextflow #bioinformatics + ${{ steps.get_topics.outputs.topics }} #nfcore #openscience #nextflow #bioinformatics send-tweet: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.10" - name: Install dependencies diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml index 3805dc81c1..e8140bfb12 100644 --- a/nf_core/pipeline-template/.nf-core.yml +++ b/nf_core/pipeline-template/.nf-core.yml @@ -1 +1,2 @@ repository_type: pipeline +nf_core_version: "{{ nf_core_version }}" diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index af57081f60..4dc0f1dcd7 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -3,6 +3,9 @@ repos: rev: "v3.1.0" hooks: - id: prettier + additional_dependencies: + - prettier@3.2.5 + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "2.7.3" hooks: diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index a3b141cf38..88e0f1719f 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -20,7 +20,7 @@ [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/{{ name }}) +[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/{{ name }}) {% endif -%} {%- if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index f73c5afaa4..9c62bf0634 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -59,7 +59,4 @@ process { errorStrategy = 'retry' maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index e3ea8fa6c4..d203d2b6e6 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -22,14 +22,6 @@ process { ext.args = '--quiet' } - withName: CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } - withName: 'MULTIQC' { ext.args = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' } publishDir = [ diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 32b9619ebb..827e21b7b7 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,7 +22,7 @@ params { // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {% if igenomes -%} // Genome references diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index d92692fa94..aa3b70c1e5 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -17,13 +17,13 @@ params { // Input data for full size test // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- else -%} // Fasta references - fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + fasta = params.pipelines_testdata_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index c908d3d38c..d46dfca04c 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -163,6 +163,8 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) +- `wave` + - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow ` 24.03.0-edge` or later). - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 38aa622063..9137c59675 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,7 +7,7 @@ "nf-core": { "fastqc": { "branch": "master", - "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", + "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["modules"] }, "multiqc": { @@ -26,7 +26,7 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 9e19a74c56..d79f1c862d 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -25,6 +25,11 @@ process FASTQC { def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // FastQC memory value allowed range (100 - 10000) + def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) + """ printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name @@ -33,6 +38,7 @@ process FASTQC { fastqc \\ $args \\ --threads $task.cpus \\ + --memory $fastqc_memory \\ $renamed_files cat <<-END_VERSIONS > versions.yml diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 17e75f18a4..7648a5ebc3 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -18,8 +18,7 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes/' igenomes_ignore = false - fasta = null - {%- endif -%} + {%- endif %} // MultiQC options multiqc_config = null @@ -29,15 +28,16 @@ params { multiqc_methods_description = null // Boilerplate options - outdir = null - publish_dir_mode = 'copy' - email = null - email_on_fail = null - plaintext_email = false - monochrome_logs = false - hook_url = null - help = false - version = false + outdir = null + publish_dir_mode = 'copy' + email = null + email_on_fail = null + plaintext_email = false + monochrome_logs = false + hook_url = null + help = false + version = false + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null @@ -77,105 +77,111 @@ try { } // Load {{ name }} custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! -// try { -// includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") -// } +try { + includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" +} catch (Exception e) { + System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") +} {% endif -%} profiles { debug { - dumpHashes = true - process.beforeScript = 'echo $HOSTNAME' - cleanup = false + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false nextflow.enable.configProcessNamesValidation = true } conda { - conda.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - channels = ['conda-forge', 'bioconda', 'defaults'] - apptainer.enabled = false + conda.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + conda.channels = ['conda-forge', 'bioconda', 'defaults'] + apptainer.enabled = false } mamba { - conda.enabled = true - conda.useMamba = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + conda.enabled = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } docker { - docker.enabled = true - conda.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' + docker.enabled = true + conda.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { - singularity.enabled = true - singularity.autoMounts = true - conda.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + singularity.enabled = true + singularity.autoMounts = true + conda.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } podman { - podman.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + podman.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } shifter { - shifter.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + shifter.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } charliecloud { - charliecloud.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - apptainer.enabled = false + charliecloud.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + apptainer.enabled = false } apptainer { - apptainer.enabled = true - apptainer.autoMounts = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + apptainer.enabled = true + apptainer.autoMounts = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + wave { + apptainer.ociAutoPull = true + singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } gitpod { - executor.name = 'local' - executor.cpus = 4 - executor.memory = 8.GB + executor.name = 'local' + executor.cpus = 4 + executor.memory = 8.GB } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 77b1a7a070..ae7c0b715f 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -267,6 +267,13 @@ "description": "Validation of parameters in lenient more.", "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." + }, + "pipelines_testdata_base_path": { + "type": "string", + "fa_icon": "far fa-check-circle", + "description": "Base URL or local path to location of pipeline test dataset files", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/", + "hidden": true } } } diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml deleted file mode 100644 index 56110621e7..0000000000 --- a/nf_core/pipeline-template/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. -# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.ruff] -line-length = 120 -target-version = "py38" -cache-dir = "~/.cache/ruff" - -[tool.ruff.lint] -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] - -[tool.ruff.lint.isort] -known-first-party = ["nf_core"] - -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402", "F401"] diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 5bc51caab8..a4bfb9f8be 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -143,6 +143,10 @@ workflow PIPELINE_COMPLETION { imNotification(summary_params, hook_url) } } + + workflow.onError { + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + } } /* @@ -239,8 +243,16 @@ def methodsDescriptionText(mqc_methods_yaml) { meta["manifest_map"] = workflow.manifest.toMap() // Pipeline DOI - meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" - meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + if (meta.manifest_map.doi) { + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + def temp_doi_ref = "" + String[] manifest_doi = meta.manifest_map.doi.tokenize(",") + for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref.replace("https://doi.org/", "").replace(" ", "")}), " + meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length() - 2) + } else meta["doi_text"] = "" + meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " // Tool references meta["tool_citations"] = "" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a8b55d6fe1..14558c3927 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -65,9 +65,15 @@ def checkProfileProvided(nextflow_cli_args) { // Citation string for pipeline // def workflowCitation() { + def temp_doi_ref = "" + String[] manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + - " ${workflow.manifest.doi}\n\n" + + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 68adbaa328..de0f21fe38 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -40,22 +40,44 @@ workflow {{ short_name|upper }} { // Collate and save software versions // softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) - .set { ch_collated_versions } + .collectFile( + storeDir: "${params.outdir}/pipeline_info", + name: 'nf_core_pipeline_software_mqc_versions.yml', + sort: true, + newLine: true + ).set { ch_collated_versions } // // MODULE: MultiQC // - ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) - ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() - ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) - ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) + ch_multiqc_config = Channel.fromPath( + "$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? + Channel.fromPath(params.multiqc_config, checkIfExists: true) : + Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? + Channel.fromPath(params.multiqc_logo, checkIfExists: true) : + Channel.empty() + + summary_params = paramsSummaryMap( + workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? + file(params.multiqc_methods_description, checkIfExists: true) : + file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value( + methodsDescriptionText(ch_multiqc_custom_methods_description)) + + ch_multiqc_files = ch_multiqc_files.mix( + ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix( + ch_methods_description.collectFile( + name: 'methods_description_mqc.yaml', + sort: true + ) + ) MULTIQC ( ch_multiqc_files.collect(), diff --git a/nf_core/schema.py b/nf_core/schema.py index df04dc5a1e..4f5acfa0af 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -1,5 +1,4 @@ -""" Code to deal with pipeline JSON Schema """ - +"""Code to deal with pipeline JSON Schema""" import copy import json @@ -7,6 +6,7 @@ import tempfile import webbrowser from pathlib import Path +from typing import Union import jinja2 import jsonschema @@ -47,11 +47,14 @@ def __init__(self): self.web_schema_build_web_url = None self.web_schema_build_api_url = None - def get_schema_path(self, path, local_only=False, revision=None): + def get_schema_path( + self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None + ) -> None: """Given a pipeline name, directory, or path, set self.schema_filename""" path = Path(path) # Supplied path exists - assume a local pipeline directory or schema if path.exists(): + log.debug(f"Path exists: {path}. Assuming local pipeline directory or schema") if revision is not None: log.warning(f"Local workflow supplied, ignoring revision '{revision}'") if path.is_dir(): @@ -64,14 +67,16 @@ def get_schema_path(self, path, local_only=False, revision=None): # Path does not exist - assume a name of a remote workflow elif not local_only: self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) - self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") - + self.schema_filename = Path(self.pipeline_dir or "", "nextflow_schema.json") + # check if the schema file exists + if not self.schema_filename.exists(): + self.schema_filename = None # Only looking for local paths, overwrite with None to be safe else: self.schema_filename = None # Check that the schema file exists - if self.schema_filename is None or not Path(self.schema_filename).exists(): + if self.schema_filename is None or not Path(self.schema_filename).exists() and local_only: error = f"Could not find pipeline schema for '{path}': {self.schema_filename}" log.error(error) raise AssertionError(error) @@ -84,16 +89,13 @@ def load_lint_schema(self): self.get_schema_defaults() self.validate_default_params() if len(self.invalid_nextflow_config_default_parameters) > 0: + params = "\n --".join( + [f"{param}: {msg}" for param, msg in self.invalid_nextflow_config_default_parameters.items()] + ) log.info( - "[red][✗] Invalid default parameters found:\n --{}\n\nNOTE: Use null in config for no default.".format( - "\n --".join( - [ - f"{param}: {msg}" - for param, msg in self.invalid_nextflow_config_default_parameters.items() - ] - ) - ) + f"[red][✗] Invalid default parameters found:\n {params} \n\nNOTE: Use null in config for no default." ) + else: log.info(f"[green][✓] Pipeline schema looks valid[/] [dim](found {num_params} params)") except json.decoder.JSONDecodeError as e: @@ -107,7 +109,7 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" - if self.schema_filename is None: + if self.schema_filename is None or not Path(self.schema_filename).exists(): raise AssertionError("Pipeline schema filename could not be found.") with open(self.schema_filename) as fh: @@ -282,9 +284,9 @@ def validate_default_params(self): if param in self.pipeline_params: self.validate_config_default_parameter(param, group_properties[param], self.pipeline_params[param]) else: - self.invalid_nextflow_config_default_parameters[ - param - ] = "Not in pipeline parameters. Check `nextflow.config`." + self.invalid_nextflow_config_default_parameters[param] = ( + "Not in pipeline parameters. Check `nextflow.config`." + ) # Go over ungrouped params if any exist ungrouped_properties = self.schema.get("properties") @@ -297,9 +299,9 @@ def validate_default_params(self): param, ungrouped_properties[param], self.pipeline_params[param] ) else: - self.invalid_nextflow_config_default_parameters[ - param - ] = "Not in pipeline parameters. Check `nextflow.config`." + self.invalid_nextflow_config_default_parameters[param] = ( + "Not in pipeline parameters. Check `nextflow.config`." + ) def validate_config_default_parameter(self, param, schema_param, config_default): """ @@ -314,9 +316,9 @@ def validate_config_default_parameter(self, param, schema_param, config_default) ): # Check that we are not deferring the execution of this parameter in the schema default with squiggly brakcets if schema_param["type"] != "string" or "{" not in schema_param["default"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Schema default (`{schema_param['default']}`) does not match the config default (`{config_default}`)" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Schema default (`{schema_param['default']}`) does not match the config default (`{config_default}`)" + ) return # if default is null, we're good @@ -326,28 +328,28 @@ def validate_config_default_parameter(self, param, schema_param, config_default) # Check variable types in nextflow.config if schema_param["type"] == "string": if str(config_default) in ["false", "true", "''"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"String should not be set to `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"String should not be set to `{config_default}`" + ) if schema_param["type"] == "boolean": if str(config_default) not in ["false", "true"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Booleans should only be true or false, not `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Booleans should only be true or false, not `{config_default}`" + ) if schema_param["type"] == "integer": try: int(config_default) except ValueError: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Does not look like an integer: `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Does not look like an integer: `{config_default}`" + ) if schema_param["type"] == "number": try: float(config_default) except ValueError: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Does not look like a number (float): `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Does not look like a number (float): `{config_default}`" + ) def validate_schema(self, schema=None): """ @@ -498,7 +500,7 @@ def print_documentation( if not output_fn: console = rich.console.Console() - console.print("\n", Syntax(prettified_docs, format), "\n") + console.print("\n", Syntax(prettified_docs, format, word_wrap=True), "\n") else: if Path(output_fn).exists() and not force: log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") @@ -585,10 +587,12 @@ def make_skeleton_schema(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) schema_template = env.get_template("nextflow_schema.json") + template_vars = { - "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent).strip("'"), + "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent.name).strip("'"), "description": self.pipeline_manifest.get("description", "").strip("'"), } + self.schema = json.loads(schema_template.render(template_vars)) self.get_schema_defaults() @@ -647,17 +651,13 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): # Extra help for people running offline if "Could not connect" in e.args[0]: log.info( - "If you're working offline, now copy your schema ({}) and paste at https://nf-co.re/pipeline_schema_builder".format( - self.schema_filename - ) + f"If you're working offline, now copy your schema ({self.schema_filename}) and paste at https://nf-co.re/pipeline_schema_builder" ) log.info("When you're finished, you can paste the edited schema back into the same file") if self.web_schema_build_web_url: log.info( "To save your work, open {}\n" - "Click the blue 'Finished' button, copy the schema and paste into this file: {}".format( - self.web_schema_build_web_url, self.schema_filename - ) + f"Click the blue 'Finished' button, copy the schema and paste into this file: { self.web_schema_build_web_url, self.schema_filename}" ) return False @@ -808,7 +808,7 @@ def add_schema_found_configs(self): p_def := self.build_schema_param(p_val).get("default") ): if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' | config: '{p_def}'). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{type(s_def)}: {s_def}' | config: '{type(p_def)}: {p_def}'). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) diff --git a/nf_core/subworkflow-template/tests/main.nf.test b/nf_core/subworkflow-template/tests/main.nf.test.j2 similarity index 100% rename from nf_core/subworkflow-template/tests/main.nf.test rename to nf_core/subworkflow-template/tests/main.nf.test.j2 diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 3a87190422..cc79ed8639 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -6,7 +6,6 @@ nf-core subworkflows lint """ - import logging import os @@ -208,6 +207,11 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Otherwise run all the lint tests else: + if self.repo_type == "pipeline" and self.modules_json: + # Set correct sha + version = self.modules_json.get_subworkflow_version(swf.component_name, swf.repo_url, swf.org) + swf.git_sha = version + for test_name in self.lint_tests: getattr(self, test_name)(swf) diff --git a/nf_core/subworkflows/lint/subworkflow_changes.py b/nf_core/subworkflows/lint/subworkflow_changes.py index b7fa13d931..a9c9616a21 100644 --- a/nf_core/subworkflows/lint/subworkflow_changes.py +++ b/nf_core/subworkflows/lint/subworkflow_changes.py @@ -1,6 +1,7 @@ """ Check whether the content of a subworkflow has changed compared to the original repository """ + from pathlib import Path import nf_core.modules.modules_repo diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index f7284320ea..cfae2d553c 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -1,6 +1,7 @@ """ Lint the tests of a subworkflow in nf-core/modules """ + import json import logging from pathlib import Path @@ -19,7 +20,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): It verifies that the test directory exists and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``. - Additionally, hecks that all included components in test ``main.nf`` are specified in ``test.yml`` + Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ repo_dir = subworkflow.component_dir.parts[ @@ -113,6 +114,22 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + subworkflow.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + subworkflow.warned.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: subworkflow.failed.append( ( diff --git a/nf_core/sync.py b/nf_core/sync.py index 995baeacd2..5e7b198d8d 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -1,5 +1,4 @@ -"""Synchronise a pipeline TEMPLATE branch with the template. -""" +"""Synchronise a pipeline TEMPLATE branch with the template.""" import json import logging @@ -45,6 +44,7 @@ class PipelineSync: gh_username (str): GitHub username gh_repo (str): GitHub repository name template_yaml_path (str): Path to template.yml file for pipeline creation settings. DEPRECATED + force_pr (bool): Force the creation of a pull request, even if there are no changes to the template Attributes: pipeline_dir (str): Path to target pipeline directory @@ -65,6 +65,7 @@ def __init__( gh_repo=None, gh_username=None, template_yaml_path=None, + force_pr=False, ): """Initialise syncing object""" @@ -77,6 +78,7 @@ def __init__( self.make_pr = make_pr self.gh_pr_returned_data = {} self.required_config_vars = ["manifest.name", "manifest.description", "manifest.version", "manifest.author"] + self.force_pr = force_pr self.gh_username = gh_username self.gh_repo = gh_repo @@ -133,8 +135,12 @@ def sync(self): self.make_template_pipeline() self.commit_template_changes() + if not self.made_changes and self.force_pr: + log.info("No changes made to TEMPLATE, but PR forced") + self.made_changes = True + # Push and make a pull request if we've been asked to - if self.made_changes and self.make_pr: + if self.made_changes and self.make_pr or self.force_pr: try: # Check that we have an API auth token if os.environ.get("GITHUB_AUTH_TOKEN", "") == "": diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 5c31e96911..4d6a3f6a4e 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -2,6 +2,7 @@ import logging import os import shutil +from configparser import NoOptionError, NoSectionError from pathlib import Path from typing import Dict @@ -116,6 +117,10 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url + self.repo = None + # TODO: SyncedRepo doesn't have this method and both the ModulesRepo and + # the WorkflowRepo define their own including custom init methods. This needs + # fixing. self.setup_local_repo(remote_url, branch, hide_progress) config_fn, repo_config = load_tools_config(self.local_repo_dir) @@ -319,13 +324,28 @@ def component_files_identical(self, component_name, base_path, commit, component component_dir = self.get_component_dir(component_name, component_type) for file in component_files: try: - files_identical[file] = filecmp.cmp(os.path.join(component_dir, file), os.path.join(base_path, file)) + files_identical[file] = filecmp.cmp(Path(component_dir, file), Path(base_path, file)) except FileNotFoundError: - log.debug(f"Could not open file: {os.path.join(component_dir, file)}") + log.debug(f"Could not open file: {Path(component_dir, file)}") continue self.checkout_branch() return files_identical + def ensure_git_user_config(self, default_name: str, default_email: str) -> None: + try: + with self.repo.config_reader() as git_config: + user_name = git_config.get_value("user", "name", default=None) + user_email = git_config.get_value("user", "email", default=None) + except (NoOptionError, NoSectionError): + user_name = user_email = None + + if not user_name or not user_email: + with self.repo.config_writer() as git_config: + if not user_name: + git_config.set_value("user", "name", default_name) + if not user_email: + git_config.set_value("user", "email", default_email) + def get_component_git_log(self, component_name, component_type, depth=None): """ Fetches the commit history the of requested module/subworkflow since a given date. The default value is diff --git a/nf_core/utils.py b/nf_core/utils.py index e1778b55b3..5b31f48f4b 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1,6 +1,7 @@ """ Common utility functions for the nf-core python package. """ + import concurrent.futures import datetime import errno @@ -179,7 +180,7 @@ def _load_pipeline_config(self): """ self.nf_config = fetch_wf_config(self.wf_path) - self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").split("/") + self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "/").strip("'").split("/") nextflow_version_match = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) if nextflow_version_match: @@ -212,10 +213,14 @@ def is_pipeline_directory(wf_path): for fn in ["main.nf", "nextflow.config"]: path = os.path.join(wf_path, fn) if not os.path.isfile(path): - raise UserWarning(f"'{wf_path}' is not a pipeline - '{fn}' is missing") + if wf_path == ".": + warning = f"Current directory is not a pipeline - '{fn}' is missing." + else: + warning = f"'{wf_path}' is not a pipeline - '{fn}' is missing." + raise UserWarning(warning) -def fetch_wf_config(wf_path, cache_config=True): +def fetch_wf_config(wf_path: str, cache_config: bool = True) -> dict: """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -235,13 +240,13 @@ def fetch_wf_config(wf_path, cache_config=True): cache_path = None # Nextflow home directory - use env var if set, or default to ~/.nextflow - nxf_home = os.environ.get("NXF_HOME", os.path.join(os.getenv("HOME"), ".nextflow")) + nxf_home = Path(os.environ.get("NXF_HOME", Path(os.getenv("HOME") or "", ".nextflow"))) # Build a cache directory if we can - if os.path.isdir(nxf_home): - cache_basedir = os.path.join(nxf_home, "nf-core") - if not os.path.isdir(cache_basedir): - os.mkdir(cache_basedir) + if nxf_home.is_dir(): + cache_basedir = Path(nxf_home, "nf-core") + if not cache_basedir.is_dir(): + cache_basedir.mkdir(parents=True, exist_ok=True) # If we're given a workflow object with a commit, see if we have a cached copy cache_fn = None @@ -249,7 +254,7 @@ def fetch_wf_config(wf_path, cache_config=True): concat_hash = "" for fn in ["nextflow.config", "main.nf"]: try: - with open(os.path.join(wf_path, fn), "rb") as fh: + with open(Path(wf_path, fn), "rb") as fh: concat_hash += hashlib.sha256(fh.read()).hexdigest() except FileNotFoundError: pass @@ -259,8 +264,8 @@ def fetch_wf_config(wf_path, cache_config=True): cache_fn = f"wf-config-cache-{bighash[:25]}.json" if cache_basedir and cache_fn: - cache_path = os.path.join(cache_basedir, cache_fn) - if os.path.isfile(cache_path) and cache_config is True: + cache_path = Path(cache_basedir, cache_fn) + if cache_path.is_file() and cache_config is True: log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path) as fh: try: @@ -285,10 +290,11 @@ def fetch_wf_config(wf_path, cache_config=True): # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. try: - main_nf = os.path.join(wf_path, "main.nf") - with open(main_nf) as fh: + main_nf = Path(wf_path, "main.nf") + with open(main_nf, "rb") as fh: for line in fh: - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=", line) + line_str = line.decode("utf-8") + match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line_str) if match: config[match.group(1)] = "null" except FileNotFoundError as e: @@ -453,6 +459,7 @@ def __init__(self): # pylint: disable=super-init-not-called self.auth_mode = None self.return_ok = [200, 201] self.return_retry = [403] + self.return_unauthorised = [401] self.has_init = False def lazy_init(self): @@ -545,6 +552,8 @@ def safe_get(self, url): raise e else: return r + elif request.status_code in self.return_unauthorised: + raise RuntimeError("GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") return request @@ -876,7 +885,7 @@ def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False): Args: wf_releases (array): Array of repo releases as returned by the GitHub API wf_branches (array): Array of repo branches, as returned by the GitHub API - multiple (bool): Allow selection of multiple releases & branches (for Tower) + multiple (bool): Allow selection of multiple releases & branches (for Seqera Platform) Returns: choice (str): Selected release / branch name @@ -1008,7 +1017,7 @@ def get_repo_releases_branches(pipeline, wfs): DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] -def load_tools_config(directory: Union[str, Path] = "."): +def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Path, dict]: """ Parse the nf-core.yml configuration file @@ -1036,7 +1045,6 @@ def load_tools_config(directory: Union[str, Path] = "."): with open(config_fn) as fh: tools_config = yaml.safe_load(fh) - # If the file is empty tools_config = tools_config or {} @@ -1046,12 +1054,13 @@ def load_tools_config(directory: Union[str, Path] = "."): def determine_base_dir(directory="."): base_dir = start_dir = Path(directory).absolute() - while base_dir != base_dir.parent: + # Only iterate up the tree if the start dir doesn't have a config + while not get_first_available_path(base_dir, CONFIG_PATHS) and base_dir != base_dir.parent: base_dir = base_dir.parent config_fn = get_first_available_path(base_dir, CONFIG_PATHS) if config_fn: - return directory if base_dir == start_dir else base_dir - return directory + break + return directory if base_dir == start_dir else base_dir def get_first_available_path(directory, paths): diff --git a/pytest.ini b/pytest.ini index 652bdf8e53..cf37159478 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,4 @@ [pytest] -filterwarnings = - ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 testpaths = tests python_files = test_*.py diff --git a/requirements.txt b/requirements.txt index e4319d2352..6b5b3ab57d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ pillow pdiff pre-commit prompt_toolkit>=3.0.3 -pytest-workflow>=1.6.0 +pytest-workflow>=2.0.0 pytest>=7.0.0 pyyaml questionary>=1.8.0 diff --git a/setup.py b/setup.py index c7f3fa41c6..1ebf4a6b38 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.13.1" +version = "2.14.0" with open("README.md") as f: readme = f.read() diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index c5067d7210..3176569ec8 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,4 +1,5 @@ """Test generate a snapshot""" + import json from pathlib import Path from unittest.mock import MagicMock @@ -119,9 +120,12 @@ def test_test_not_found(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) + test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test") + test_file.rename(test_file.parent / "main.nf.test.bak") with pytest.raises(UserWarning) as e: snap_generator.run() assert "Test file 'main.nf.test' not found" in str(e.value) + Path(test_file.parent / "main.nf.test.bak").rename(test_file) def test_unstable_snapshot(self): diff --git a/tests/components/snapshot_test.py b/tests/components/snapshot_test.py index d774618476..b3fc259770 100644 --- a/tests/components/snapshot_test.py +++ b/tests/components/snapshot_test.py @@ -1,4 +1,5 @@ """Test the 'modules test' or 'subworkflows test' command which runs nf-test test.""" + import shutil from pathlib import Path diff --git a/tests/lint/actions_schema_validation.py b/tests/lint/actions_schema_validation.py index ad65d90018..e202b3b1ce 100644 --- a/tests/lint/actions_schema_validation.py +++ b/tests/lint/actions_schema_validation.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import yaml @@ -9,10 +9,11 @@ def test_actions_schema_validation_missing_jobs(self): """Missing 'jobs' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: + awstest_yml_path = Path(new_pipeline) / ".github" / "workflows" / "awstest.yml" + with open(awstest_yml_path) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml.pop("jobs") - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + with open(awstest_yml_path, "w") as fh: yaml.dump(awstest_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -27,10 +28,11 @@ def test_actions_schema_validation_missing_on(self): """Missing 'on' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: + awstest_yml_path = Path(new_pipeline) / ".github" / "workflows" / "awstest.yml" + with open(awstest_yml_path) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml.pop(True) - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + with open(awstest_yml_path, "w") as fh: yaml.dump(awstest_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -38,7 +40,7 @@ def test_actions_schema_validation_missing_on(self): results = lint_obj.actions_schema_validation() - assert results["failed"][0] == "Missing 'on' keyword in {}.format(wf)" + assert results["failed"][0] == "Missing 'on' keyword in awstest.yml" assert "Workflow validation failed for awstest.yml: 'on' is a required property" in results["failed"][1] @@ -46,10 +48,11 @@ def test_actions_schema_validation_fails_for_additional_property(self): """Missing 'jobs' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: + awstest_yml_path = Path(new_pipeline) / ".github" / "workflows" / "awstest.yml" + with open(awstest_yml_path) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml["not_jobs"] = awstest_yml["jobs"] - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + with open(awstest_yml_path, "w") as fh: yaml.dump(awstest_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) diff --git a/tests/lint/configs.py b/tests/lint/configs.py new file mode 100644 index 0000000000..b50a1393aa --- /dev/null +++ b/tests/lint/configs.py @@ -0,0 +1,89 @@ +from pathlib import Path + +import yaml + +import nf_core.create +import nf_core.lint + + +def test_withname_in_modules_config(self): + """Tests finding withName in modules.config passes linting.""" + + new_pipeline = self._make_pipeline_copy() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 0 + assert any( + ["`FASTQC` found in `conf/modules.config` and Nextflow scripts." in passed for passed in result["passed"]] + ) + + +def test_superfluous_withname_in_modules_config_fails(self): + """Tests finding withName in modules.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to modules.config + modules_config = Path(new_pipeline) / "conf" / "modules.config" + with open(modules_config, "a") as f: + f.write("\nwithName: 'BPIPE' {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline, hide_progress=False) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/modules.config` contains `withName:BPIPE`") + + +def test_ignore_modules_config(self): + """Tests ignoring the modules.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore modules.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"modules_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + Path(new_pipeline, "conf", "modules.config").unlink() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/modules.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) + + +def test_superfluous_withname_in_base_config_fails(self): + """Tests finding withName in base.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to base.config + base_config = Path(new_pipeline) / "conf" / "base.config" + with open(base_config, "a") as f: + f.write("\nwithName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/base.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + + +def test_ignore_base_config(self): + """Tests ignoring the base.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore base.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"base_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + Path(new_pipeline, "conf", "base.config").unlink() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/base.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) diff --git a/tests/lint/multiqc_config.py b/tests/lint/multiqc_config.py index 721560ce81..463d5e7654 100644 --- a/tests/lint/multiqc_config.py +++ b/tests/lint/multiqc_config.py @@ -5,14 +5,37 @@ import nf_core.lint -def test_multiqc_config_exists_ignore(self): +def test_multiqc_config_exists(self): """Test that linting fails if the multiqc_config.yml file is missing""" # Delete the file new_pipeline = self._make_pipeline_copy() Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() result = lint_obj.multiqc_config() - assert result["ignored"] == ["'assets/multiqc_config.yml' not found"] + assert result["failed"] == ["`assets/multiqc_config.yml` not found."] + + +def test_multiqc_config_ignore(self): + """Test that linting succeeds if the multiqc_config.yml file is missing but ignored""" + # Delete the file + new_pipeline = self._make_pipeline_copy() + Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() + with open(Path(new_pipeline, ".nf-core.yml")) as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"multiqc_config": False} + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(content, f) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.multiqc_config() + assert result["ignored"] == ["`assets/multiqc_config.yml` not found, but it is ignored."] + + # cleanup + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(old_content, f) def test_multiqc_config_missing_report_section_order(self): @@ -20,7 +43,7 @@ def test_multiqc_config_missing_report_section_order(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml.pop("report_section_order") with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -30,7 +53,7 @@ def test_multiqc_config_missing_report_section_order(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain `report_section_order`"] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain `report_section_order`"] def test_multiqc_incorrect_export_plots(self): @@ -38,7 +61,7 @@ def test_multiqc_incorrect_export_plots(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["export_plots"] = False with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -48,7 +71,7 @@ def test_multiqc_incorrect_export_plots(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain 'export_plots: true'."] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain 'export_plots: true'."] def test_multiqc_config_report_comment_fail(self): @@ -56,7 +79,7 @@ def test_multiqc_config_report_comment_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["report_comment"] = "This is a test" with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -67,7 +90,7 @@ def test_multiqc_config_report_comment_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_fail(self): @@ -75,7 +98,7 @@ def test_multiqc_config_report_comment_release_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -87,7 +110,7 @@ def test_multiqc_config_report_comment_release_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_succeed(self): @@ -103,4 +126,4 @@ def test_multiqc_config_report_comment_release_succeed(self): # lint again lint_obj._load() result = lint_obj.multiqc_config() - assert "'assets/multiqc_config.yml' contains a matching 'report_comment'." in result["passed"] + assert "`assets/multiqc_config.yml` contains a matching 'report_comment'." in result["passed"] diff --git a/tests/lint/nextflow_config.py b/tests/lint/nextflow_config.py index fa85568f14..06af8c4fb8 100644 --- a/tests/lint/nextflow_config.py +++ b/tests/lint/nextflow_config.py @@ -83,7 +83,6 @@ def test_default_values_fail(self): with open(nf_schema_file) as f: content = f.read() fail_content = re.sub(r'"default": "128.GB"', '"default": "18.GB"', content) - print(fail_content) with open(nf_schema_file, "w") as f: f.write(fail_content) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -100,6 +99,36 @@ def test_default_values_fail(self): ) +def test_catch_params_assignment_in_main_nf(self): + """Test linting fails if main.nf contains an assignment to a parameter from nextflow_schema.json.""" + new_pipeline = self._make_pipeline_copy() + # Add parameter assignment in main.nf + main_nf_file = Path(new_pipeline) / "main.nf" + with open(main_nf_file, "a") as f: + f.write("params.max_time = 42") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 1 + assert ( + result["failed"][0] + == "Config default value incorrect: `params.max_time` is set as `240.h` in `nextflow_schema.json` but is `null` in `nextflow.config`." + ) + + +def test_allow_params_reference_in_main_nf(self): + """Test linting allows for references like `params.aligner == 'bwa'` in main.nf. The test will detect if the bug mentioned in GitHub-issue #2833 reemerges.""" + new_pipeline = self._make_pipeline_copy() + # Add parameter reference in main.nf + main_nf_file = Path(new_pipeline) / "main.nf" + with open(main_nf_file, "a") as f: + f.write("params.max_time == 42") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 0 + + def test_default_values_ignored(self): """Test ignoring linting of default values.""" new_pipeline = self._make_pipeline_copy() diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py new file mode 100644 index 0000000000..474ccd48fc --- /dev/null +++ b/tests/lint/nfcore_yml.py @@ -0,0 +1,53 @@ +import re +from pathlib import Path + +import nf_core.create +import nf_core.lint + + +def test_nfcore_yml_pass(self): + """Lint test: nfcore_yml - PASS""" + self.lint_obj._load() + results = self.lint_obj.nfcore_yml() + + assert "Repository type in `.nf-core.yml` is valid" in str(results["passed"]) + assert "nf-core version in `.nf-core.yml` is set to the latest version" in str(results["passed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_repo_type(self): + """Lint test: nfcore_yml - FAIL - repository type not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = content.replace("repository_type: pipeline", "repository_type: foo") + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "Repository type in `.nf-core.yml` is not valid." in str(results["failed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_nfcore_version(self): + """Lint test: nfcore_yml - FAIL - nf-core version not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = re.sub(r"nf_core_version:.+", "nf_core_version: foo", content) + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "nf-core version in `.nf-core.yml` is not set to the latest version." in str(results["warned"]) + assert len(results.get("failed", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 9bd280ddd8..595509de4e 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -1,3 +1,4 @@ +import json from pathlib import Path import pytest @@ -563,30 +564,29 @@ def test_modules_missing_required_tag(self): def test_modules_missing_tags_yml(self): """Test linting a module with a missing tags.yml file""" - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak") - ) + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") - ) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_tags_yml_exists" + # cleanup + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") def test_modules_incorrect_tags_yml_key(self): """Test linting a module with an incorrect key in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("bpipe/test:", "bpipe_test:") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=True, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 @@ -596,14 +596,15 @@ def test_modules_incorrect_tags_yml_key(self): def test_modules_incorrect_tags_yml_values(self): """Test linting a module with an incorrect path in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("modules/nf-core/bpipe/test/**", "foo") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 @@ -642,3 +643,68 @@ def test_nftest_failing_linting(self): assert module_lint.failed[2].lint_test == "test_main_tags" assert "kallisto/index" in module_lint.failed[2].message assert module_lint.failed[3].lint_test == "test_tags_yml" + + +def test_modules_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap")) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_versions" + + +def test_modules_empty_file_in_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_modules_empty_file_in_stub_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in module_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index b89b7b78ce..73d2452b37 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -1,3 +1,5 @@ +import json +import shutil from pathlib import Path import pytest @@ -181,3 +183,160 @@ def test_subworkflows_lint_capitalization_fail(self): # cleanup self.subworkflow_remove.remove("bam_stats_samtools", force=True) + + +def test_subworkflows_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + with open(snap_file) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open(snap_file, "w") as fh: + fh.write(new_content) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" + assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) + + # cleanup + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_missing_test_dir(self): + """Test linting a nf-test subworkflow if the tests directory is missing""" + test_dir = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests") + test_dir_copy = shutil.copytree(test_dir, test_dir.parent / "tests_copy") + shutil.rmtree(test_dir) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" + assert any([x.lint_test == "test_dir_versions" for x in subworkflow_lint.warned]) + + # cleanup + shutil.copytree(test_dir_copy, test_dir) + + +def test_subworkflows_missing_main_nf(self): + """Test linting a nf-test subworkflow if the main.nf file is missing""" + main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf") + main_nf_copy = shutil.copy(main_nf, main_nf.parent / "main_nf_copy") + main_nf.unlink() + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "main_nf_exists" + + # cleanup + shutil.copy(main_nf_copy, main_nf) + + +def test_subworkflows_empty_file_in_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_empty_file_in_stub_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in subworkflow_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_missing_tags_yml(self): + """Test linting a subworkflow with a missing tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) >= 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_tags_yml_exists" + + # cleanup + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") + + +def test_subworkflows_incorrect_tags_yml_key(self): + """Test linting a subworkflow with an incorrect key in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("test_subworkflow:", "subworkflow:") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=True, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) + + +def test_subworkflows_incorrect_tags_yml_values(self): + """Test linting a subworkflow with an incorrect path in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("subworkflows/nf-core/test_subworkflow/**", "foo") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 658a2339d4..c697d34287 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -1,5 +1,5 @@ -"""Some tests covering the bump_version code. -""" +"""Some tests covering the bump_version code.""" + import os import yaml diff --git a/tests/test_cli.py b/tests/test_cli.py index 1261e3a9e9..5df0a32418 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,4 @@ -""" Tests covering the command-line code. +"""Tests covering the command-line code. Most tests check the cli arguments are passed along and that some action is taken. @@ -31,13 +31,13 @@ def test_header_outdated(mock_check_outdated, mock_nf_core_cli, capsys): class TestCli(unittest.TestCase): - """Class for testing the commandline interface""" + """Class for testing the command line interface""" def setUp(self): self.runner = CliRunner() def assemble_params(self, params): - """Assemble a dictionnary of parameters into a list of arguments for the cli + """Assemble a dictionary of parameters into a list of arguments for the cli Note: if the value of a parameter is None, it will be considered a flag. @@ -166,8 +166,9 @@ def test_cli_download(self, mock_dl): "outdir": "/path/outdir", "compress": "tar.gz", "force": None, - "tower": None, + "platform": None, "download-configuration": None, + "tag": "3.12=testing", "container-system": "singularity", "container-library": "quay.io", "container-cache-utilisation": "copy", @@ -186,8 +187,9 @@ def test_cli_download(self, mock_dl): params["outdir"], params["compress"], "force" in params, - "tower" in params, + "platform" in params, "download-configuration" in params, + (params["tag"],), params["container-system"], (params["container-library"],), params["container-cache-utilisation"], @@ -363,18 +365,21 @@ def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] - result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("nextflow_schema.json") - assert "nextflow_schema.json" in result.output + with self.runner.isolated_filesystem(): + with open("nextflow_schema.json", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("nextflow_schema.json") @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" cmd = ["schema", "lint", "some_other_filename"] - result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("some_other_filename") - assert "some_other_filename" in result.output - assert "nextflow_schema.json" not in result.output + with self.runner.isolated_filesystem(): + with open("some_other_filename", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("some_other_filename") @mock.patch("nf_core.create_logo.create_logo") def test_create_logo(self, mock_create_logo): diff --git a/tests/test_components.py b/tests/test_components.py index b7f67eb51d..eaf999c3c3 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -1,5 +1,4 @@ -""" Tests covering the modules commands -""" +"""Tests covering the modules commands""" import os import shutil diff --git a/tests/test_create.py b/tests/test_create.py index 1cc073cb54..e2672499cd 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,5 +1,5 @@ -"""Some tests covering the pipeline creation sub command. -""" +"""Some tests covering the pipeline creation sub command.""" + import os import unittest from pathlib import Path diff --git a/tests/test_download.py b/tests/test_download.py index d823040247..3e0f11d579 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,12 +1,13 @@ -"""Tests for the download subcommand of nf-core tools -""" +"""Tests for the download subcommand of nf-core tools""" +import logging import os import re import shutil import tempfile import unittest from pathlib import Path +from typing import List from unittest import mock import pytest @@ -21,6 +22,24 @@ class DownloadTest(unittest.TestCase): + @pytest.fixture(autouse=True) + def use_caplog(self, caplog): + self._caplog = caplog + + @property + def logged_levels(self) -> List[str]: + return [record.levelname for record in self._caplog.records] + + @property + def logged_messages(self) -> List[str]: + return [record.message for record in self._caplog.records] + + def __contains__(self, item: str) -> bool: + """Allows to check for log messages easily using the in operator inside a test: + assert 'my log message' in self + """ + return any(record.message == item for record in self._caplog.records if self._caplog) + # # Tests for 'get_release_hash' # @@ -564,16 +583,16 @@ def test_download_workflow_with_success(self, tmp_dir, mock_download_image, mock download_obj.download_workflow() # - # Test Download for Tower + # Test Download for Seqera Platform # @with_temporary_folder @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") - def test_download_workflow_for_tower(self, tmp_dir, _): + def test_download_workflow_for_platform(self, tmp_dir, _): download_obj = DownloadWorkflow( pipeline="nf-core/rnaseq", revision=("3.7", "3.9"), compress_type="none", - tower=True, + platform=True, container_system="singularity", ) @@ -593,7 +612,7 @@ def test_download_workflow_for_tower(self, tmp_dir, _): download_obj.get_revision_hash() - # download_obj.wf_download_url is not set for tower downloads, but the sha values are + # download_obj.wf_download_url is not set for Seqera Platform downloads, but the sha values are assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 2 assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 @@ -601,7 +620,7 @@ def test_download_workflow_for_tower(self, tmp_dir, _): assert bool(re.search(r"nf-core-rnaseq_\d{4}-\d{2}-\d{1,2}_\d{1,2}-\d{1,2}", download_obj.outdir, re.S)) download_obj.output_filename = f"{download_obj.outdir}.git" - download_obj.download_workflow_tower(location=tmp_dir) + download_obj.download_workflow_platform(location=tmp_dir) assert download_obj.workflow_repo assert isinstance(download_obj.workflow_repo, WorkflowRepo) @@ -615,7 +634,7 @@ def test_download_workflow_for_tower(self, tmp_dir, _): # assert that the download has a "latest" branch. assert "latest" in all_heads - # download_obj.download_workflow_tower(location=tmp_dir) will run container image detection for all requested revisions + # download_obj.download_workflow_platform(location=tmp_dir) will run container image detection for all requested revisions assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 33 assert ( "https://depot.galaxyproject.org/singularity/bbmap:38.93--he522d1c_0" in download_obj.containers @@ -624,3 +643,88 @@ def test_download_workflow_for_tower(self, tmp_dir, _): "https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:59cdd445419f14abac76b31dd0d71217994cbcc9-0" in download_obj.containers ) # indirect definition via $container variable. + + # + # Brief test adding a single custom tag to Seqera Platform download + # + @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") + @with_temporary_folder + def test_download_workflow_for_platform_with_one_custom_tag(self, _, tmp_dir): + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=("3.9=cool_revision",), + ) + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 1 + + # + # Test adding custom tags to Seqera Platform download (full test) + # + @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") + @with_temporary_folder + def test_download_workflow_for_platform_with_custom_tags(self, _, tmp_dir): + with self._caplog.at_level(logging.INFO): + from git.refs.tag import TagReference + + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.7", "3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=( + "3.7=a.tad.outdated", + "3.9=cool_revision", + "3.9=invalid tag", + "3.14.0=not_included", + "What is this?", + ), + ) + + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + + assert isinstance(download_obj.revision, list) and len(download_obj.revision) == 2 + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 0 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 5 + + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) + + download_obj.get_revision_hash() + download_obj.output_filename = f"{download_obj.outdir}.git" + download_obj.download_workflow_platform(location=tmp_dir) + + assert download_obj.workflow_repo + assert isinstance(download_obj.workflow_repo, WorkflowRepo) + assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + assert "Locally cached repository: nf-core/rnaseq, revisions 3.7, 3.9" in repr(download_obj.workflow_repo) + + # assert that every additional tag has been passed on to the WorkflowRepo instance + assert download_obj.additional_tags == download_obj.workflow_repo.additional_tags + + # assert that the additional tags are all TagReference objects + assert all(isinstance(tag, TagReference) for tag in download_obj.workflow_repo.tags) + + workflow_repo_tags = {tag.name for tag in download_obj.workflow_repo.tags} + assert len(workflow_repo_tags) == 4 + # the invalid/malformed additional_tags should not have been added. + assert all(tag in workflow_repo_tags for tag in {"3.7", "a.tad.outdated", "cool_revision", "3.9"}) + assert not any(tag in workflow_repo_tags for tag in {"invalid tag", "not_included", "What is this?"}) + + assert all( + log in self.logged_messages + for log in { + "[red]Could not apply invalid `--tag` specification[/]: '3.9=invalid tag'", + "[red]Adding tag 'not_included' to '3.14.0' failed.[/]\n Mind that '3.14.0' must be a valid git reference that resolves to a commit.", + "[red]Could not apply invalid `--tag` specification[/]: 'What is this?'", + } + ) diff --git a/tests/test_launch.py b/tests/test_launch.py index dc8d6b147c..79dbe3fb97 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -1,5 +1,4 @@ -""" Tests covering the pipeline launch code. -""" +"""Tests covering the pipeline launch code.""" import json import os diff --git a/tests/test_licenses.py b/tests/test_licenses.py index 4fb58a107c..8023c9e891 100644 --- a/tests/test_licenses.py +++ b/tests/test_licenses.py @@ -1,5 +1,4 @@ -"""Some tests covering the pipeline creation sub command. -""" +"""Some tests covering the pipeline creation sub command.""" # import json # import os # import tempfile diff --git a/tests/test_lint.py b/tests/test_lint.py index 9839265892..b72a6bfdfa 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -1,5 +1,5 @@ -"""Some tests covering the linting code. -""" +"""Some tests covering the linting code.""" + import fnmatch import json import os @@ -198,6 +198,13 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_jobs, test_actions_schema_validation_missing_on, ) + from .lint.configs import ( # type: ignore[misc] + test_ignore_base_config, + test_ignore_modules_config, + test_superfluous_withname_in_base_config_fails, + test_superfluous_withname_in_modules_config_fails, + test_withname_in_modules_config, + ) from .lint.files_exist import ( # type: ignore[misc] test_files_exist_depreciated_file, test_files_exist_fail_conditional, @@ -213,7 +220,8 @@ def test_sphinx_md_files(self): from .lint.merge_markers import test_merge_markers_found # type: ignore[misc] from .lint.modules_json import test_modules_json_pass # type: ignore[misc] from .lint.multiqc_config import ( # type: ignore[misc] - test_multiqc_config_exists_ignore, + test_multiqc_config_exists, + test_multiqc_config_ignore, test_multiqc_config_missing_report_section_order, test_multiqc_config_report_comment_fail, test_multiqc_config_report_comment_release_fail, @@ -221,6 +229,8 @@ def test_sphinx_md_files(self): test_multiqc_incorrect_export_plots, ) from .lint.nextflow_config import ( # type: ignore[misc] + test_allow_params_reference_in_main_nf, + test_catch_params_assignment_in_main_nf, test_default_values_fail, test_default_values_float, test_default_values_float_fail, @@ -231,6 +241,11 @@ def test_sphinx_md_files(self): test_nextflow_config_example_pass, test_nextflow_config_missing_test_profile_failed, ) + from .lint.nfcore_yml import ( # type: ignore[misc] + test_nfcore_yml_fail_nfcore_version, + test_nfcore_yml_fail_repo_type, + test_nfcore_yml_pass, + ) from .lint.template_strings import ( # type: ignore[misc] test_template_strings, test_template_strings_ignore_file, diff --git a/tests/test_list.py b/tests/test_list.py index c1f51e03e0..c78276b41d 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -1,5 +1,4 @@ -""" Tests covering the workflow listing code. -""" +"""Tests covering the workflow listing code.""" import json import os diff --git a/tests/test_modules.py b/tests/test_modules.py index 539d3dcc57..d3d99abadd 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,6 +1,6 @@ -""" Tests covering the modules commands -""" +"""Tests covering the modules commands""" +import json import os import shutil import unittest @@ -54,9 +54,22 @@ def create_modules_repo_dummy(tmp_dir): yaml.dump(meta_yml, fh) # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") - test_snap_path.touch() + with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") @@ -180,6 +193,9 @@ def test_modulesrepo_class(self): test_modules_install_trimgalore_twice, ) from .modules.lint import ( # type: ignore[misc] + test_modules_absent_version, + test_modules_empty_file_in_snapshot, + test_modules_empty_file_in_stub_snapshot, test_modules_environment_yml_file_doesnt_exists, test_modules_environment_yml_file_name_mismatch, test_modules_environment_yml_file_not_array, diff --git a/tests/test_refgenie.py b/tests/test_refgenie.py index 5440c1c477..23cc0dd14a 100644 --- a/tests/test_refgenie.py +++ b/tests/test_refgenie.py @@ -1,5 +1,4 @@ -""" Tests covering the refgenie integration code -""" +"""Tests covering the refgenie integration code""" import os import shlex diff --git a/tests/test_schema.py b/tests/test_schema.py index 89fcc98b66..e0921908d4 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,11 +1,11 @@ -""" Tests covering the pipeline schema code. -""" +"""Tests covering the pipeline schema code.""" import json import os import shutil import tempfile import unittest +from pathlib import Path from unittest import mock import pytest @@ -217,6 +217,15 @@ def test_make_skeleton_schema(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema["title"] == "nf-core/test pipeline parameters" + + def test_make_skeleton_schema_absent_name(self): + """Test making a new schema skeleton""" + self.schema_obj.schema_filename = self.template_schema + self.schema_obj.pipeline_manifest["description"] = "Test pipeline" + self.schema_obj.make_skeleton_schema() + self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema["title"] == "wf pipeline parameters" def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" @@ -315,9 +324,9 @@ def test_build_schema_from_scratch(self, tmp_dir): Pretty much a copy of test_launch.py test_make_pipeline_schema """ - test_pipeline_dir = os.path.join(tmp_dir, "wf") + test_pipeline_dir = Path(tmp_dir, "wf") shutil.copytree(self.template_dir, test_pipeline_dir) - os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) + Path(test_pipeline_dir, "nextflow_schema.json").unlink() self.schema_obj.build_schema(test_pipeline_dir, True, False, None) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index cd0bd21146..0a9224002a 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,6 +1,6 @@ -""" Tests covering the subworkflows commands -""" +"""Tests covering the subworkflows commands""" +import json import os import shutil import unittest @@ -24,7 +24,6 @@ def create_modules_repo_dummy(tmp_dir): root_dir = Path(tmp_dir, "modules") Path(root_dir, "modules").mkdir(parents=True, exist_ok=True) - Path(root_dir, "subworkflows").mkdir(parents=True, exist_ok=True) Path(root_dir, "subworkflows", "nf-core").mkdir(parents=True, exist_ok=True) Path(root_dir, "tests", "config").mkdir(parents=True, exist_ok=True) with open(Path(root_dir, ".nf-core.yml"), "w") as fh: @@ -35,9 +34,22 @@ def create_modules_repo_dummy(tmp_dir): # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") - test_snap_path.touch() + test_snap_path.parent.mkdir(parents=True, exist_ok=True) with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) return root_dir @@ -120,6 +132,11 @@ def tearDown(self): test_subworkflows_install_tracking_added_super_subworkflow, ) from .subworkflows.lint import ( # type: ignore[misc] + test_subworkflows_absent_version, + test_subworkflows_empty_file_in_snapshot, + test_subworkflows_empty_file_in_stub_snapshot, + test_subworkflows_incorrect_tags_yml_key, + test_subworkflows_incorrect_tags_yml_values, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, @@ -132,6 +149,7 @@ def tearDown(self): test_subworkflows_lint_snapshot_file, test_subworkflows_lint_snapshot_file_missing_fail, test_subworkflows_lint_snapshot_file_not_needed, + test_subworkflows_missing_tags_yml, ) from .subworkflows.list import ( # type: ignore[misc] test_subworkflows_install_and_list_subworkflows, diff --git a/tests/test_sync.py b/tests/test_sync.py index 51a27653ab..b94968cd4c 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,5 +1,4 @@ -""" Tests covering the sync command -""" +"""Tests covering the sync command""" import json import os @@ -27,7 +26,12 @@ def setUp(self): self.pipeline_dir = os.path.join(self.tmp_dir, "testpipeline") default_branch = "master" self.create_obj = nf_core.create.PipelineCreate( - "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True, default_branch=default_branch + "testing", + "test pipeline", + "tester", + outdir=self.pipeline_dir, + plain=True, + default_branch=default_branch, ) self.create_obj.init_pipeline() self.remote_path = os.path.join(self.tmp_dir, "remote_repo") @@ -375,7 +379,7 @@ def test_close_open_pr(self, mock_patch, mock_post): } assert psync.close_open_pr(pr) - assert mock_patch.called_once_with("url_to_update_pr") + mock_patch.assert_called_once_with(url="url_to_update_pr", data='{"state": "closed"}') @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) @@ -398,7 +402,7 @@ def test_close_open_pr_fail(self, mock_patch, mock_post): } assert not psync.close_open_pr(pr) - assert mock_patch.called_once_with("bad_url_to_update_pr") + mock_patch.assert_called_once_with(url="bad_url_to_update_pr", data='{"state": "closed"}') def test_reset_target_dir(self): """Try resetting target pipeline directory""" diff --git a/tests/test_utils.py b/tests/test_utils.py index 4b5ab19fce..145060450f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,4 @@ -""" Tests covering for utility functions. -""" +"""Tests covering for utility functions.""" import os import shutil