Merge pull request #163 from scilus/feat/nf_core_remote

[IMPROV] Quality of Life improvments for the devcontainer
scilus · Jun 19, 2024 · defffee · defffee
2 parents 5a8a54e + 37856b2
commit defffee
Show file tree

Hide file tree

Showing 11 changed files with 154 additions and 77 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,6 +1,9 @@
-FROM scilus/scilus:1.6.0
+FROM scilus/scilus:2.0.2
+
+ARG NFTEST_VERSION
 
 ENV NODE_MAJOR=18
+ENV NFTEST_VERSION=${NFTEST_VERSION:-0.9.0-rc1}
 
 RUN apt update && apt install -y \
         curl \
@@ -15,10 +18,3 @@ RUN python3 -m pip install pipx && \
 
 RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - &&\
     apt-get install -y nodejs
-
-WORKDIR /
-RUN mkdir -p nf-test/bin
-
-WORKDIR /nf-test/bin/
-RUN curl -fsSL https://code.askimed.com/install/nf-test | bash
-ENV PATH=$PATH:/nf-test/bin
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,8 +1,14 @@
 {
     "name": "NF-SCIL development container",
-    "build": { "dockerfile": "Dockerfile" },
+    "build": {
+        "dockerfile": "Dockerfile",
+        "args": {
+            "NFTEST_VERSION": "0.9.0-rc1"
+        }
+    },
     "forwardPorts": [3000],
-    "onCreateCommand": "bash .devcontainer/setup_container.sh",
+    "onCreateCommand": "bash .devcontainer/onCreateCommand.sh",
+    "updateContentCommand": "bash .devcontainer/updateContentCommand.sh",
     "postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
     "features": {
         "ghcr.io/devcontainers/features/git:1": {},
@@ -18,48 +24,67 @@
     "mounts": [
         {
             "source": "nf-scil-venv",
-            "target": "/root/.cache/pypoetry/virtualenvs/",
+            "target": "${containerWorkspaceFolder}/.venv",
+            "type": "volume"
+        },
+        {
+            "source": "nf-scil-nf-test",
+            "target": "${containerWorkspaceFolder}/.nf-test",
             "type": "volume"
         },
         {
-            "source": "nf-scil-tmp",
             "target": "/tmp",
             "type": "volume"
+        },
+        {
+            "source": "nf-scil-profile",
+            "target": "/root",
+            "type": "volume"
+        },
+        {
+            "target": "/root/.vscode-server",
+            "type": "volume"
         }
     ],
     "customizations": {
         "vscode": {
             "settings": {
-                "pythonIndent.keepHangingBracketOnLine": true,
-                "pythonIndent.trimLinesWithOnlyWhitespace": true,
-                "python.createEnvironment.trigger": "off",
-                "python.terminal.activateEnvironment": true,
-                "python.poetryPath": "/root/.local/bin/poetry",
-                "python.defaultInterpreterPath": "/root/.cache/pypoetry/virtualenvs/",
                 "python.analysis.exclude": [
-                    "${workspaceFolder}/.dvc",
-                    "${workspaceFolder}/.pytest_cache",
-                    "${workspaceFolder}/.test_data",
-                    "${workspaceFolder}/.vscode",
-                    "${workspaceFolder}/nf-scil-extensions",
+                    "${containerWorkspaceFolder}/.dvc",
+                    "${containerWorkspaceFolder}/.pytest_cache",
+                    "${containerWorkspaceFolder}/.test_data",
+                    "${containerWorkspaceFolder}/.vscode",
+                    "${containerWorkspaceFolder}/.venv",
+                    "${containerWorkspaceFolder}/nf-scil-extensions",
                     "**/__pycache__",
-                    "${workspaceFolder}/.git"
+                    "${containerWorkspaceFolder}/.git"
                 ],
                 "python.analysis.ignore": [
-                    "${workspaceFolder}/.dvc",
-                    "${workspaceFolder}/.pytest_cache",
-                    "${workspaceFolder}/.test_data",
-                    "${workspaceFolder}/.vscode",
-                    "${workspaceFolder}/nf-scil-extensions",
+                    "${containerWorkspaceFolder}/.dvc",
+                    "${containerWorkspaceFolder}/.pytest_cache",
+                    "${containerWorkspaceFolder}/.test_data",
+                    "${containerWorkspaceFolder}/.vscode",
+                    "${containerWorkspaceFolder}/.venv",
+                    "${containerWorkspaceFolder}/nf-scil-extensions",
                     "**/__pycache__",
-                    "${workspaceFolder}/.git"
-                ]
+                    "${containerWorkspaceFolder}/.git"
+                ],
+                "python.createEnvironment.trigger": "off",
+                "python.interpreter.infoVisibility": "always",
+                "python.poetryPath": "poetry",
+                "python.defaultInterpreterPath": "${containerWorkspaceFolder}/.venv",
+                "python.terminal.activateEnvironment": true,
+                "python.terminal.activateEnvInCurrentTerminal": true,
+                "python.terminal.focusAfterLaunch": true,
+                "pythonIndent.keepHangingBracketOnLine": true,
+                "pythonIndent.trimLinesWithOnlyWhitespace": true
             },
             "extensions": [
                 "AlexVCaron.nf-scil-extensions",
                 "ms-python.autopep8",
                 "ms-python.vscode-pylance",
-                "mutantdino.resourcemonitor"
+                "mutantdino.resourcemonitor",
+                "KorbinianEckstein.niivue"
             ]
         }
     },

diff --git a/.devcontainer/onCreateCommand.sh b/.devcontainer/onCreateCommand.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+npm install -g --save-dev --save-exact prettier
+npm install -g editorconfig
+npm install -g --save-dev editorconfig-checker
+
+mkdir -p $XDG_CONFIG_HOME/nf-scil
+touch $XDG_CONFIG_HOME/nf-scil/.env
+echo "source $XDG_CONFIG_HOME/nf-scil/.env" >> ~/.bashrc
+
+mkdir -p /nf-test/bin
+cd /nf-test/bin/
+curl -fsSL https://code.askimed.com/install/nf-test | bash -s ${NFTEST_VERSION}
+echo "PATH=$PATH:/nf-test/bin" >> ~/.bashrc
diff --git a/.devcontainer/setup_container.sh b/.devcontainer/setup_container.sh
diff --git a/.devcontainer/updateContentCommand.sh b/.devcontainer/updateContentCommand.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+
+GIT_REMOTE=$(git remote get-url origin)
+# Get tracked remote branch associated to current branch (default to main)
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "main")
+
+cat <<EOF > $XDG_CONFIG_HOME/nf-scil/.env
+# This file is used to store environment variables for the project.
+# It is sourced by the shell on startup of every terminals.
+
+export PROFILE=docker
+export NFCORE_MODULES_GIT_REMOTE=$GIT_REMOTE
+export NFCORE_MODULES_BRANCH=$CURRENT_BRANCH
+export NFCORE_SUBWORKFLOWS_GIT_REMOTE=$GIT_REMOTE
+export NFCORE_SUBWORKFLOWS_BRANCH=$CURRENT_BRANCH
+EOF
+
+poetry install --no-root
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,6 +1,5 @@
 {
     "editor.detectIndentation": false,
-    "python.languageServer": "Pylance",
     "python.missingPackage.severity": "Warning",
     "python.terminal.activateEnvInCurrentTerminal": true,
     "python.terminal.focusAfterLaunch": true,
@@ -11,11 +10,6 @@
     "python.analysis.exclude": ["**/__pycache__"],
     "python.analysis.ignore": ["**/__pycache__"],
     "python.analysis.importFormat": "relative",
-    "python.analysis.logLevel": "Warning",
-    "git.allowForcePush": true,
-    "git.confirmForcePush": true,
-    "git.useForcePushIfIncludes": true,
-    "git.useForcePushWithLease": true,
     "git.branchProtection": ["main"],
     "git.branchProtectionPrompt": "alwaysPrompt",
     "github.branchProtection": true,
@@ -123,6 +117,5 @@
         "voxels",
         "wmparc",
         "xform"
-    ],
-    "markdown.extension.toc.updateOnSave": false
+    ]
 }
diff --git a/README.md b/README.md
@@ -47,6 +47,16 @@ framework.
     - [Test data infrastructure](docs/MODULE.md#test-data-infrastructure)
       - [Using the .test_data directory](docs/MODULE.md#using-the-test_data-directory)
       - [Using Scilpy Fetcher](docs/MODULE.md#using-scilpy-fetcher)
+    - [Adding a new subworkflow to nf-scil](docs/SUBWORKFLOWS.md#adding-a-new-subworkflow-to-nf-scil)
+      - [Generate the template](docs/SUBWORKFLOWS.md#generate-the-template)
+      - [Generate the template](docs/SUBWORKFLOWS.md#generate-the-template-1)
+        - [Edit the subworkflow's main.nf](docs/SUBWORKFLOWS.md#edit-subworkflowsnf-scilname_of_your_workflowmainnf)
+          - [Define your Subworkflow inputs.](docs/SUBWORKFLOWS.md#define-your-subworkflow-inputs)
+          - [Fill the `main:` section.](docs/SUBWORKFLOWS.md#fill-the-main-section)
+          - [define your Workflow outputs.](docs/SUBWORKFLOWS.md#define-your-workflow-outputs)
+        - [Edit the subworkflow's meta.yml](docs/SUBWORKFLOWS.md#edit-subworkflowsnf-scilname_of_your_workflowmetayml)
+      - [Lint your code](docs/SUBWORKFLOWS.md#lint-your-code)
+      - [Submit your PR](docs/SUBWORKFLOWS.md#submit-your-pr)
   - [Running tests](#running-tests)
   - [Configuring Docker for easy usage](#configuring-docker-for-easy-usage)
   - [Installing Prettier](#installing-prettier)
@@ -72,6 +82,25 @@ nf-core modules \
   list remote
 ```
 
+The same can be done for subworkflows, replacing `modules` in the `nf-core` command by `subworkflows, e.g. :
+
+```bash
+nf-core subworkflows \
+  --git-remote https://github.com/scilus/nf-scil.git \
+  install <category>/<subworkflow>
+```
+
+It can become heavy to always prepend the commands with `--git-remote`, even so if you need to specify a `--branch` where to fetch the information. You can instead define the `git-remote` and `branch` using _Environment Variables_ :
+
+```bash
+export NFCORE_MODULES_GIT_REMOTE=https://github.com/scilus/nf-scil.git
+export NFCORE_MODULES_GIT_BRANCH=main
+export NFCORE_SUBWORKFLOWS_GIT_REMOTE=https://github.com/scilus/nf-scil.git
+export NFCORE_SUBWORKFLOWS_GIT_BRANCH=main
+```
+
+and call all commands without specifying the `--git-remote` and `--branch` options, while still targeting the `nf-scil` repository.
+
 # Developing in `nf-scil`
 
 The `nf-scil` project requires some specific tools to be installed on your system so that the development environment runs correctly. You can [install them manually](#manual-configuration), but if you desire to streamline the process and start coding faster, we highly recommend using the [VS Code development container](#configuration-via-the-devcontainer) to get fully configured in a matter of minutes.
@@ -85,6 +114,7 @@ The `nf-scil` project requires some specific tools to be installed on your syste
 - Java Runtime &geq; 11, &leq; 17
   - On Ubuntu, install `openjdk-jre-<version>` packages
 - Nextflow &geq; 21.04.3
+- nf-test &geq; 0.9.0-rc1
 - Node &geq; 14 and Prettier (see [below](#installing-prettier))
 
 > [!IMPORTANT]
@@ -149,6 +179,19 @@ To exit the environment, simply enter the `exit` command in the shell.
 > the environment gracefully, making it so you won't be able to reactivate it without
 > exiting the shell.
 
+### Global environment
+
+Set the following environment variables in your `.bashrc` (or whatever is the equivalent for your shell) :
+
+```bash
+export NFCORE_MODULES_GIT_REMOTE=https://github.com/scilus/nf-scil.git
+export NFCORE_MODULES_GIT_BRANCH=main
+export NFCORE_SUBWORKFLOWS_GIT_REMOTE=https://github.com/scilus/nf-scil.git
+export NFCORE_SUBWORKFLOWS_GIT_BRANCH=main
+```
+
+This will make it so the `nf-core` commands target the right repository by default. Else, you'll need to add `--git-remote` and `--branch` options to pretty much all commands relating to `modules` and `subworkflows`.
+
 ### Working with VS Code
 
 The `nf-scil` project curates a bundle of useful extensions for Visual Studio Code, the `nf-scil-extensions` package. You can find it easily on the [extension
@@ -167,9 +210,10 @@ environment.
 
   - `nf-scil`, `nf-core` all accessible through the terminal, which is configured to load
     the `poetry` environment in shells automatically
+  - `nf-scil` configured as the main repository for all `nf-core` commands, using `NFCORE_*` environment variables
   - `git`, `git-lfs`, `github-cli`
   - `curl`, `wget`, `apt-get`
-  - `nextflow`, `docker`, `tmux`
+  - `nextflow`, `nf-test`, `docker`, `tmux`
 
 - Available in the VS Code IDE through extensions :
   - Docker images and containers management
@@ -179,20 +223,14 @@ environment.
 
 ## Contributing to the `nf-scil` project
 
-If you want to propose a new `module` to the repository, follow the guidelines in the
-[module creation](./docs/MODULE.md) documentation. We follow standards closely
-aligned with `nf-core`, with some exceptions on process atomicity and how test data is
-handled. Modules that don't abide to them won't be accepted and PR containing them will
-be closed automatically.
+If you want to propose a new `module` to the repository, follow the guidelines in the [module creation](./docs/MODULE.md) documentation. The same goes for `subworkflows`, using [these guidelines](./docs/SUBWORKFLOWS.md) instead. We follow standards closely aligned with `nf-core`, with some exceptions on process atomicity and how test data is handled. Modules that don't abide to them won't be accepted and PR containing them will be closed automatically.
 
 ## Running tests
 
 Tests are run through `nf-core`, using the command :
 
 ```bash
-nf-core modules \
-  --git-remote https://github.com/scilus/nf-scil.git \
-  test <category/tool>
+nf-core modules test <category/tool>
 ```
 
 The tool can be omitted to run tests for all modules in a category.

diff --git a/docs/MODULE.md b/docs/MODULE.md
@@ -165,10 +165,7 @@ don't need to specify them all. At least define the `keywords`, describe the pro
 Once done, commit your module and push the changes. Then, to look at the documentation it creates for your module, run :
 
 ```bash
-nf-core modules \
-  --git-remote <your reository> \
-  --branch <your branch unless main branch> \
-  info <category/name>
+nf-core modules info <category/name>
 ```
 
 ### Editing `./tests/modules/nf-scil/<category>/<tool>/main.nf` :
@@ -213,11 +210,7 @@ so output files that gets generated are checksum correctly.
 Run :
 
 ```bash
-nf-core modules create-test-yml \
-    --run-tests \
-    --force \
-    --no-prompts \
-    <category>/<tool>
+nf-core modules create-test-yml --run-tests --force --no-prompts <category>/<tool>
 ```
 
 All the test case you defined will be run, watch out for errors ! Once everything runs
@@ -230,10 +223,7 @@ critical to ensure future executions of your test produce valid outputs.
 Before submitting to _nf-scil_, once you've commit and push everything, the code need to be correctly linted, else the checks won't pass. This is done using `prettier` on your new module, through the _nf-core_ command line :
 
 ```bash
-nf-core modules \
-  --git-remote <your repository> \
-  --branch <your branch unless main branch> \
-  lint <category>/<tool>
+nf-core modules lint <category>/<tool>
 ```
 
 You'll probably get a bunch of _whitespace_ and _indentation_ errors, but also image errors, bad _nextflow_ syntax and more. You need to fix all `errors` and as much as the `ẁarnings`as possible.

diff --git a/docs/SUBWORKFLOWS.md b/docs/SUBWORKFLOWS.md
@@ -1,5 +1,16 @@
 # Adding a new subworkflow to nf-scil
 
+- [Adding a new subworkflow to nf-scil](#adding-a-new-subworkflow-to-nf-scil)
+  - [Generate the template](#generate-the-template)
+  - [Generate the template](#generate-the-template-1)
+    - [Edit the subworkflow's main.nf](#edit-subworkflowsnf-scilname_of_your_workflowmainnf)
+      - [Define your Subworkflow inputs.](#define-your-subworkflow-inputs)
+      - [Fill the `main:` section.](#fill-the-main-section)
+      - [define your Workflow outputs.](#define-your-workflow-outputs)
+    - [Edit the subworkflow's meta.yml](#edit-subworkflowsnf-scilname_of_your_workflowmetayml)
+  - [Lint your code](#lint-your-code)
+  - [Submit your PR](#submit-your-pr)
+
 ## Generate the template
 
 First verify you are located at the root of this repository (not in `subworkflows`), then run the following interactive command :
@@ -116,7 +127,7 @@ emit:
 Don't forget to also define the output for the version file :
 
 ```
-    versions = ch_versions // channel: [ versions.yml ]
+versions = ch_versions // channel: [ versions.yml ]
 ```
 
 ### Edit `./subworkflows/nf-scil/<name_of_your_workflow>/meta.yml`
@@ -129,10 +140,7 @@ don't need to specify them all. Provide at least 3 relevant `keywords` and list
 Run `prettier` on your new module, through the `nf-core` command line :
 
 ```
-  nf-core subworkflows \
-    --git-remote <your repository> \
-    --branch <your branch unless main branch> \
-    lint <subworkflow>
+nf-core subworkflows lint <subworkflow>
 ```
 
 and fix all `errors` and as many `warnings` as possible.

diff --git a/nf-test.config b/nf-test.config
@@ -10,4 +10,6 @@ config {
 
     // run all test with the defined docker profile from the main nextflow.config
     profile ""
-}
+
+    ignore [".venv/*"]
+}
diff --git a/poetry.toml b/poetry.toml
@@ -1,2 +1,2 @@
 [virtualenvs]
-in-project = false
+in-project = true