diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f1a89953..624e73ea 100755 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,9 @@ -FROM scilus/scilus:1.6.0 +FROM scilus/scilus:2.0.2 + +ARG NFTEST_VERSION ENV NODE_MAJOR=18 +ENV NFTEST_VERSION=${NFTEST_VERSION:-0.9.0-rc1} RUN apt update && apt install -y \ curl \ @@ -15,10 +18,3 @@ RUN python3 -m pip install pipx && \ RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - &&\ apt-get install -y nodejs - -WORKDIR / -RUN mkdir -p nf-test/bin - -WORKDIR /nf-test/bin/ -RUN curl -fsSL https://code.askimed.com/install/nf-test | bash -ENV PATH=$PATH:/nf-test/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ad241ca5..6412e4d1 100755 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,8 +1,14 @@ { "name": "NF-SCIL development container", - "build": { "dockerfile": "Dockerfile" }, + "build": { + "dockerfile": "Dockerfile", + "args": { + "NFTEST_VERSION": "0.9.0-rc1" + } + }, "forwardPorts": [3000], - "onCreateCommand": "bash .devcontainer/setup_container.sh", + "onCreateCommand": "bash .devcontainer/onCreateCommand.sh", + "updateContentCommand": "bash .devcontainer/updateContentCommand.sh", "postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}", "features": { "ghcr.io/devcontainers/features/git:1": {}, @@ -18,48 +24,67 @@ "mounts": [ { "source": "nf-scil-venv", - "target": "/root/.cache/pypoetry/virtualenvs/", + "target": "${containerWorkspaceFolder}/.venv", + "type": "volume" + }, + { + "source": "nf-scil-nf-test", + "target": "${containerWorkspaceFolder}/.nf-test", "type": "volume" }, { - "source": "nf-scil-tmp", "target": "/tmp", "type": "volume" + }, + { + "source": "nf-scil-profile", + "target": "/root", + "type": "volume" + }, + { + "target": "/root/.vscode-server", + "type": "volume" } ], "customizations": { "vscode": { "settings": { - "pythonIndent.keepHangingBracketOnLine": true, - "pythonIndent.trimLinesWithOnlyWhitespace": true, - "python.createEnvironment.trigger": "off", - "python.terminal.activateEnvironment": true, - "python.poetryPath": "/root/.local/bin/poetry", - "python.defaultInterpreterPath": "/root/.cache/pypoetry/virtualenvs/", "python.analysis.exclude": [ - "${workspaceFolder}/.dvc", - "${workspaceFolder}/.pytest_cache", - "${workspaceFolder}/.test_data", - "${workspaceFolder}/.vscode", - "${workspaceFolder}/nf-scil-extensions", + "${containerWorkspaceFolder}/.dvc", + "${containerWorkspaceFolder}/.pytest_cache", + "${containerWorkspaceFolder}/.test_data", + "${containerWorkspaceFolder}/.vscode", + "${containerWorkspaceFolder}/.venv", + "${containerWorkspaceFolder}/nf-scil-extensions", "**/__pycache__", - "${workspaceFolder}/.git" + "${containerWorkspaceFolder}/.git" ], "python.analysis.ignore": [ - "${workspaceFolder}/.dvc", - "${workspaceFolder}/.pytest_cache", - "${workspaceFolder}/.test_data", - "${workspaceFolder}/.vscode", - "${workspaceFolder}/nf-scil-extensions", + "${containerWorkspaceFolder}/.dvc", + "${containerWorkspaceFolder}/.pytest_cache", + "${containerWorkspaceFolder}/.test_data", + "${containerWorkspaceFolder}/.vscode", + "${containerWorkspaceFolder}/.venv", + "${containerWorkspaceFolder}/nf-scil-extensions", "**/__pycache__", - "${workspaceFolder}/.git" - ] + "${containerWorkspaceFolder}/.git" + ], + "python.createEnvironment.trigger": "off", + "python.interpreter.infoVisibility": "always", + "python.poetryPath": "poetry", + "python.defaultInterpreterPath": "${containerWorkspaceFolder}/.venv", + "python.terminal.activateEnvironment": true, + "python.terminal.activateEnvInCurrentTerminal": true, + "python.terminal.focusAfterLaunch": true, + "pythonIndent.keepHangingBracketOnLine": true, + "pythonIndent.trimLinesWithOnlyWhitespace": true }, "extensions": [ "AlexVCaron.nf-scil-extensions", "ms-python.autopep8", "ms-python.vscode-pylance", - "mutantdino.resourcemonitor" + "mutantdino.resourcemonitor", + "KorbinianEckstein.niivue" ] } }, diff --git a/.devcontainer/onCreateCommand.sh b/.devcontainer/onCreateCommand.sh new file mode 100755 index 00000000..011dd957 --- /dev/null +++ b/.devcontainer/onCreateCommand.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +npm install -g --save-dev --save-exact prettier +npm install -g editorconfig +npm install -g --save-dev editorconfig-checker + +mkdir -p $XDG_CONFIG_HOME/nf-scil +touch $XDG_CONFIG_HOME/nf-scil/.env +echo "source $XDG_CONFIG_HOME/nf-scil/.env" >> ~/.bashrc + +mkdir -p /nf-test/bin +cd /nf-test/bin/ +curl -fsSL https://code.askimed.com/install/nf-test | bash -s ${NFTEST_VERSION} +echo "PATH=$PATH:/nf-test/bin" >> ~/.bashrc diff --git a/.devcontainer/setup_container.sh b/.devcontainer/setup_container.sh deleted file mode 100755 index 438fa810..00000000 --- a/.devcontainer/setup_container.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -poetry install --no-root -echo "export PROFILE=docker" >> ~/.bashrc - -npm install -g --save-dev --save-exact prettier -npm install -g editorconfig -npm install -g --save-dev editorconfig-checker diff --git a/.devcontainer/updateContentCommand.sh b/.devcontainer/updateContentCommand.sh new file mode 100644 index 00000000..051e660e --- /dev/null +++ b/.devcontainer/updateContentCommand.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + + +GIT_REMOTE=$(git remote get-url origin) +# Get tracked remote branch associated to current branch (default to main) +CURRENT_BRANCH=$(git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "main") + +cat < $XDG_CONFIG_HOME/nf-scil/.env +# This file is used to store environment variables for the project. +# It is sourced by the shell on startup of every terminals. + +export PROFILE=docker +export NFCORE_MODULES_GIT_REMOTE=$GIT_REMOTE +export NFCORE_MODULES_BRANCH=$CURRENT_BRANCH +export NFCORE_SUBWORKFLOWS_GIT_REMOTE=$GIT_REMOTE +export NFCORE_SUBWORKFLOWS_BRANCH=$CURRENT_BRANCH +EOF + +poetry install --no-root diff --git a/.vscode/settings.json b/.vscode/settings.json index afd3033c..373002e7 100755 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,5 @@ { "editor.detectIndentation": false, - "python.languageServer": "Pylance", "python.missingPackage.severity": "Warning", "python.terminal.activateEnvInCurrentTerminal": true, "python.terminal.focusAfterLaunch": true, @@ -11,11 +10,6 @@ "python.analysis.exclude": ["**/__pycache__"], "python.analysis.ignore": ["**/__pycache__"], "python.analysis.importFormat": "relative", - "python.analysis.logLevel": "Warning", - "git.allowForcePush": true, - "git.confirmForcePush": true, - "git.useForcePushIfIncludes": true, - "git.useForcePushWithLease": true, "git.branchProtection": ["main"], "git.branchProtectionPrompt": "alwaysPrompt", "github.branchProtection": true, @@ -123,6 +117,5 @@ "voxels", "wmparc", "xform" - ], - "markdown.extension.toc.updateOnSave": false + ] } diff --git a/README.md b/README.md index 1acaffc3..628edd2b 100755 --- a/README.md +++ b/README.md @@ -47,6 +47,16 @@ framework. - [Test data infrastructure](docs/MODULE.md#test-data-infrastructure) - [Using the .test_data directory](docs/MODULE.md#using-the-test_data-directory) - [Using Scilpy Fetcher](docs/MODULE.md#using-scilpy-fetcher) + - [Adding a new subworkflow to nf-scil](docs/SUBWORKFLOWS.md#adding-a-new-subworkflow-to-nf-scil) + - [Generate the template](docs/SUBWORKFLOWS.md#generate-the-template) + - [Generate the template](docs/SUBWORKFLOWS.md#generate-the-template-1) + - [Edit the subworkflow's main.nf](docs/SUBWORKFLOWS.md#edit-subworkflowsnf-scilname_of_your_workflowmainnf) + - [Define your Subworkflow inputs.](docs/SUBWORKFLOWS.md#define-your-subworkflow-inputs) + - [Fill the `main:` section.](docs/SUBWORKFLOWS.md#fill-the-main-section) + - [define your Workflow outputs.](docs/SUBWORKFLOWS.md#define-your-workflow-outputs) + - [Edit the subworkflow's meta.yml](docs/SUBWORKFLOWS.md#edit-subworkflowsnf-scilname_of_your_workflowmetayml) + - [Lint your code](docs/SUBWORKFLOWS.md#lint-your-code) + - [Submit your PR](docs/SUBWORKFLOWS.md#submit-your-pr) - [Running tests](#running-tests) - [Configuring Docker for easy usage](#configuring-docker-for-easy-usage) - [Installing Prettier](#installing-prettier) @@ -72,6 +82,25 @@ nf-core modules \ list remote ``` +The same can be done for subworkflows, replacing `modules` in the `nf-core` command by `subworkflows, e.g. : + +```bash +nf-core subworkflows \ + --git-remote https://github.com/scilus/nf-scil.git \ + install / +``` + +It can become heavy to always prepend the commands with `--git-remote`, even so if you need to specify a `--branch` where to fetch the information. You can instead define the `git-remote` and `branch` using _Environment Variables_ : + +```bash +export NFCORE_MODULES_GIT_REMOTE=https://github.com/scilus/nf-scil.git +export NFCORE_MODULES_GIT_BRANCH=main +export NFCORE_SUBWORKFLOWS_GIT_REMOTE=https://github.com/scilus/nf-scil.git +export NFCORE_SUBWORKFLOWS_GIT_BRANCH=main +``` + +and call all commands without specifying the `--git-remote` and `--branch` options, while still targeting the `nf-scil` repository. + # Developing in `nf-scil` The `nf-scil` project requires some specific tools to be installed on your system so that the development environment runs correctly. You can [install them manually](#manual-configuration), but if you desire to streamline the process and start coding faster, we highly recommend using the [VS Code development container](#configuration-via-the-devcontainer) to get fully configured in a matter of minutes. @@ -85,6 +114,7 @@ The `nf-scil` project requires some specific tools to be installed on your syste - Java Runtime ≥ 11, ≤ 17 - On Ubuntu, install `openjdk-jre-` packages - Nextflow ≥ 21.04.3 +- nf-test ≥ 0.9.0-rc1 - Node ≥ 14 and Prettier (see [below](#installing-prettier)) > [!IMPORTANT] @@ -149,6 +179,19 @@ To exit the environment, simply enter the `exit` command in the shell. > the environment gracefully, making it so you won't be able to reactivate it without > exiting the shell. +### Global environment + +Set the following environment variables in your `.bashrc` (or whatever is the equivalent for your shell) : + +```bash +export NFCORE_MODULES_GIT_REMOTE=https://github.com/scilus/nf-scil.git +export NFCORE_MODULES_GIT_BRANCH=main +export NFCORE_SUBWORKFLOWS_GIT_REMOTE=https://github.com/scilus/nf-scil.git +export NFCORE_SUBWORKFLOWS_GIT_BRANCH=main +``` + +This will make it so the `nf-core` commands target the right repository by default. Else, you'll need to add `--git-remote` and `--branch` options to pretty much all commands relating to `modules` and `subworkflows`. + ### Working with VS Code The `nf-scil` project curates a bundle of useful extensions for Visual Studio Code, the `nf-scil-extensions` package. You can find it easily on the [extension @@ -167,9 +210,10 @@ environment. - `nf-scil`, `nf-core` all accessible through the terminal, which is configured to load the `poetry` environment in shells automatically + - `nf-scil` configured as the main repository for all `nf-core` commands, using `NFCORE_*` environment variables - `git`, `git-lfs`, `github-cli` - `curl`, `wget`, `apt-get` - - `nextflow`, `docker`, `tmux` + - `nextflow`, `nf-test`, `docker`, `tmux` - Available in the VS Code IDE through extensions : - Docker images and containers management @@ -179,20 +223,14 @@ environment. ## Contributing to the `nf-scil` project -If you want to propose a new `module` to the repository, follow the guidelines in the -[module creation](./docs/MODULE.md) documentation. We follow standards closely -aligned with `nf-core`, with some exceptions on process atomicity and how test data is -handled. Modules that don't abide to them won't be accepted and PR containing them will -be closed automatically. +If you want to propose a new `module` to the repository, follow the guidelines in the [module creation](./docs/MODULE.md) documentation. The same goes for `subworkflows`, using [these guidelines](./docs/SUBWORKFLOWS.md) instead. We follow standards closely aligned with `nf-core`, with some exceptions on process atomicity and how test data is handled. Modules that don't abide to them won't be accepted and PR containing them will be closed automatically. ## Running tests Tests are run through `nf-core`, using the command : ```bash -nf-core modules \ - --git-remote https://github.com/scilus/nf-scil.git \ - test +nf-core modules test ``` The tool can be omitted to run tests for all modules in a category. diff --git a/docs/MODULE.md b/docs/MODULE.md index 7a634323..8abb51e6 100755 --- a/docs/MODULE.md +++ b/docs/MODULE.md @@ -165,10 +165,7 @@ don't need to specify them all. At least define the `keywords`, describe the pro Once done, commit your module and push the changes. Then, to look at the documentation it creates for your module, run : ```bash -nf-core modules \ - --git-remote \ - --branch \ - info +nf-core modules info ``` ### Editing `./tests/modules/nf-scil///main.nf` : @@ -213,11 +210,7 @@ so output files that gets generated are checksum correctly. Run : ```bash -nf-core modules create-test-yml \ - --run-tests \ - --force \ - --no-prompts \ - / +nf-core modules create-test-yml --run-tests --force --no-prompts / ``` All the test case you defined will be run, watch out for errors ! Once everything runs @@ -230,10 +223,7 @@ critical to ensure future executions of your test produce valid outputs. Before submitting to _nf-scil_, once you've commit and push everything, the code need to be correctly linted, else the checks won't pass. This is done using `prettier` on your new module, through the _nf-core_ command line : ```bash -nf-core modules \ - --git-remote \ - --branch \ - lint / +nf-core modules lint / ``` You'll probably get a bunch of _whitespace_ and _indentation_ errors, but also image errors, bad _nextflow_ syntax and more. You need to fix all `errors` and as much as the `ẁarnings`as possible. diff --git a/docs/SUBWORKFLOWS.md b/docs/SUBWORKFLOWS.md index 44913285..bc997631 100644 --- a/docs/SUBWORKFLOWS.md +++ b/docs/SUBWORKFLOWS.md @@ -1,5 +1,16 @@ # Adding a new subworkflow to nf-scil +- [Adding a new subworkflow to nf-scil](#adding-a-new-subworkflow-to-nf-scil) + - [Generate the template](#generate-the-template) + - [Generate the template](#generate-the-template-1) + - [Edit the subworkflow's main.nf](#edit-subworkflowsnf-scilname_of_your_workflowmainnf) + - [Define your Subworkflow inputs.](#define-your-subworkflow-inputs) + - [Fill the `main:` section.](#fill-the-main-section) + - [define your Workflow outputs.](#define-your-workflow-outputs) + - [Edit the subworkflow's meta.yml](#edit-subworkflowsnf-scilname_of_your_workflowmetayml) + - [Lint your code](#lint-your-code) + - [Submit your PR](#submit-your-pr) + ## Generate the template First verify you are located at the root of this repository (not in `subworkflows`), then run the following interactive command : @@ -116,7 +127,7 @@ emit: Don't forget to also define the output for the version file : ``` - versions = ch_versions // channel: [ versions.yml ] +versions = ch_versions // channel: [ versions.yml ] ``` ### Edit `./subworkflows/nf-scil//meta.yml` @@ -129,10 +140,7 @@ don't need to specify them all. Provide at least 3 relevant `keywords` and list Run `prettier` on your new module, through the `nf-core` command line : ``` - nf-core subworkflows \ - --git-remote \ - --branch \ - lint +nf-core subworkflows lint ``` and fix all `errors` and as many `warnings` as possible. diff --git a/nf-test.config b/nf-test.config index 966ab7d6..048040f4 100755 --- a/nf-test.config +++ b/nf-test.config @@ -10,4 +10,6 @@ config { // run all test with the defined docker profile from the main nextflow.config profile "" -} \ No newline at end of file + + ignore [".venv/*"] +} diff --git a/poetry.toml b/poetry.toml index c0c340ac..ab1033bd 100755 --- a/poetry.toml +++ b/poetry.toml @@ -1,2 +1,2 @@ [virtualenvs] -in-project = false +in-project = true