nf-core download: Merge ability to add custom tags to revisions in do…

…wnloaded pipelines

CHANGELOG.md

@@ -1,6 +1,6 @@
 # nf-core/tools: Changelog
 
-## v2.13.2dev
+## v2.14.0dev
 
 ### Template
 
@@ -27,8 +27,9 @@
 
 ### Download
 
-- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release.
+- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. ([#2853](https://github.com/nf-core/tools/pull/2853))
 - Better error message when GITHUB_TOKEN exists but is wrong/outdated
+- New `--tag` argument to add custom tags during a pipeline download ([#2938](https://github.com/nf-core/tools/pull/2938))
 
 ### Components
 

README.md

@@ -437,6 +437,9 @@ Subsequently, the `*.git` folder can be moved to it's final destination and link
 > [!TIP]
 > Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified.
 
+Facilities and those who are setting up pipelines for others to use may find the `--tag` argument helpful. It allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `--tag "3.12.0=testing" --tag "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production.
+The `--tag` argument must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository.
+
 ## Pipeline software licences
 
 Sometimes it's useful to see the software licences of the tools used in a pipeline.

nf_core/__main__.py

@@ -368,14 +368,14 @@ def create_params_file(pipeline, revision, output, force, show_hidden):
 @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files")
 # TODO: Remove this in a future release. Deprecated in March 2024.
 @click.option(
+    "-t",
     "--tower",
     is_flag=True,
     default=False,
     hidden=True,
-    help="Download for Seqera Platform. DEPRECATED: Please use --platform instead.",
+    help="Download for Seqera Platform. DEPRECATED: Please use `--platform` instead.",
 )
 @click.option(
-    "-t",
     "--platform",
     is_flag=True,
     default=False,
@@ -388,6 +388,11 @@ def create_params_file(pipeline, revision, output, force, show_hidden):
     default=False,
     help="Include configuration profiles in download. Not available with `--platform`",
 )
+@click.option(
+    "--tag",
+    multiple=True,
+    help="Add custom alias tags to `--platform` downloads. For example, `--tag \"3.10=validated\"` adds the custom 'validated' tag to the 3.10 release.",
+)
 # -c changed to -s for consistency with other --container arguments, where it is always the first letter of the last word.
 # Also -c might be used instead of -d for config in a later release, but reusing params for different options in two subsequent releases might be too error-prone.
 @click.option(
@@ -430,6 +435,7 @@ def download(
     tower,
     platform,
     download_configuration,
+    tag,
     container_system,
     container_library,
     container_cache_utilisation,
@@ -444,6 +450,9 @@ def download(
     """
     from nf_core.download import DownloadWorkflow
 
+    if tower:
+        log.warning("[red]The `-t` / `--tower` flag is deprecated. Please use `--platform` instead.[/]")
+
     dl = DownloadWorkflow(
         pipeline,
         revision,
@@ -452,6 +461,7 @@ def download(
         force,
         tower or platform,  # True if either specified
         download_configuration,
+        tag,
         container_system,
         container_library,
         container_cache_utilisation,

nf_core/download.py

@@ -88,10 +88,18 @@ class DownloadWorkflow:
 
     Args:
         pipeline (str): A nf-core pipeline name.
-        revision (List[str]): The workflow revision to download, like `1.0`. Defaults to None.
-        container (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False.
-        platform (bool): Flag, to customize the download for Seqera Platform (convert to git bare repo). Defaults to False.
+        revision (List[str]): The workflow revision(s) to download, like `1.0` or `dev` . Defaults to None.
         outdir (str): Path to the local download directory. Defaults to None.
+        compress_type (str): Type of compression for the downloaded files. Defaults to None.
+        force (bool): Flag to force download even if files already exist (overwrite existing files). Defaults to False.
+        platform (bool): Flag to customize the download for Seqera Platform (convert to git bare repo). Defaults to False.
+        download_configuration (str): Download the configuration files from nf-core/configs. Defaults to None.
+        tag (List[str]): Specify additional tags to add to the downloaded pipeline. Defaults to None.
+        container_system (str): The container system to use (e.g., "singularity"). Defaults to None.
+        container_library (List[str]): The container libraries (registries) to use. Defaults to None.
+        container_cache_utilisation (str): If a local or remote cache of already existing container images should be considered. Defaults to None.
+        container_cache_index (str): An index for the remote container cache. Defaults to None.
+        parallel_downloads (int): The number of parallel downloads to use. Defaults to 4.
     """
 
     def __init__(
@@ -103,6 +111,7 @@ def __init__(
         force=False,
         platform=False,
         download_configuration=None,
+        additional_tags=None,
         container_system=None,
         container_library=None,
         container_cache_utilisation=None,
@@ -125,6 +134,15 @@ def __init__(
         # this implies that non-interactive "no" choice is only possible implicitly (e.g. with --platform or if prompt is suppressed by !stderr.is_interactive).
         # only alternative would have been to make it a parameter with argument, e.g. -d="yes" or -d="no".
         self.include_configs = True if download_configuration else False if bool(platform) else None
+        # Additional tags to add to the downloaded pipeline. This enables to mark particular commits or revisions with
+        # additional tags, e.g. "stable", "testing", "validated", "production" etc. Since this requires a git-repo, it is only
+        # available for the bare / Seqera Platform download.
+        if isinstance(additional_tags, str) and bool(len(additional_tags)) and self.platform:
+            self.additional_tags = [additional_tags]
+        elif isinstance(additional_tags, tuple) and bool(len(additional_tags)) and self.platform:
+            self.additional_tags = [*additional_tags]
+        else:
+            self.additional_tags = None
         # Specifying a cache index or container library implies that containers should be downloaded.
         self.container_system = "singularity" if container_cache_index or bool(container_library) else container_system
         # Manually specified container library (registry)
@@ -282,6 +300,7 @@ def download_workflow_platform(self, location=None):
             remote_url=f"https://github.com/{self.pipeline}.git",
             revision=self.revision if self.revision else None,
             commit=self.wf_sha.values() if bool(self.wf_sha) else None,
+            additional_tags=self.additional_tags,
             location=(location if location else None),  # manual location is required for the tests to work
             in_cache=False,
         )
@@ -1489,6 +1508,7 @@ def __init__(
         remote_url,
         revision,
         commit,
+        additional_tags,
         location=None,
         hide_progress=False,
         in_cache=True,
@@ -1523,13 +1543,21 @@ def __init__(
 
         self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache)
 
-        # expose some instance attributes
-        self.tags = self.repo.tags
+        # additional tags to be added to the repository
+        self.additional_tags = additional_tags if additional_tags else None
 
     def __repr__(self):
         """Called by print, creates representation of object"""
         return f"<Locally cached repository: {self.fullname}, revisions {', '.join(self.revision)}\n cached at: {self.local_repo_dir}>"
 
+    @property
+    def heads(self):
+        return self.repo.heads
+
+    @property
+    def tags(self):
+        return self.repo.tags
+
     def access(self):
         if os.path.exists(self.local_repo_dir):
             return self.local_repo_dir
@@ -1640,7 +1668,6 @@ def tidy_tags_and_branches(self):
                 # delete unwanted tags from repository
                 for tag in tags_to_remove:
                     self.repo.delete_tag(tag)
-                self.tags = self.repo.tags
 
                 # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master")
                 self.checkout(self.revision[0])
@@ -1677,7 +1704,8 @@ def tidy_tags_and_branches(self):
                     if self.repo.head.is_detached:
                         self.repo.head.reset(index=True, working_tree=True)
 
-                self.heads = self.repo.heads
+                # Apply the custom additional tags to the repository
+                self.__add_additional_tags()
 
                 # get all tags and available remote_branches
                 completed_revisions = {revision.name for revision in self.repo.heads + self.repo.tags}
@@ -1695,6 +1723,39 @@ def tidy_tags_and_branches(self):
                 self.retry_setup_local_repo(skip_confirm=True)
                 raise DownloadError(e) from e
 
+    # "Private" method to add the additional custom tags to the repository.
+    def __add_additional_tags(self) -> None:
+        if self.additional_tags:
+            # example.com is reserved by the Internet Assigned Numbers Authority (IANA)  as special-use domain names for documentation purposes.
+            # Although "dev-null" is a syntactically-valid local-part that is equally valid for delivery,
+            # and only the receiving MTA can decide whether to accept it, it is to my best knowledge configured with
+            # a Postfix discard mail delivery agent (https://www.postfix.org/discard.8.html), so incoming mails should be sinkholed.
+            self.ensure_git_user_config(f"nf-core download v{nf_core.__version__}", "dev-null@example.com")
+
+            for additional_tag in self.additional_tags:
+                # A valid git branch or tag name can contain alphanumeric characters, underscores, hyphens, and dots.
+                # But it must not start with a dot, hyphen or underscore and also cannot contain two consecutive dots.
+                if re.match(r"^\w[\w_.-]+={1}\w[\w_.-]+$", additional_tag) and ".." not in additional_tag:
+                    anchor, tag = additional_tag.split("=")
+                    if self.repo.is_valid_object(anchor) and not self.repo.is_valid_object(tag):
+                        try:
+                            self.repo.create_tag(
+                                tag, ref=anchor, message=f"Synonynmous tag to {anchor}; added by `nf-core download`."
+                            )
+                        except (GitCommandError, InvalidGitRepositoryError) as e:
+                            log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n")
+                    else:
+                        if not self.repo.is_valid_object(anchor):
+                            log.error(
+                                f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit."
+                            )
+                        if self.repo.is_valid_object(tag):
+                            log.error(
+                                f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{tag}' must not exist hitherto."
+                            )
+                else:
+                    log.error(f"[red]Could not apply invalid `--tag` specification[/]: '{additional_tag}'")
+
     def bare_clone(self, destination):
         if self.repo:
             try:

nf_core/synced_repo.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import shutil
+from configparser import NoOptionError, NoSectionError
 from pathlib import Path
 from typing import Dict
 
@@ -116,6 +117,10 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa
 
         self.remote_url = remote_url
 
+        self.repo = None
+        # TODO: SyncedRepo doesn't have this method and both the ModulesRepo and
+        # the WorkflowRepo define their own including custom init methods. This needs
+        # fixing.
         self.setup_local_repo(remote_url, branch, hide_progress)
 
         config_fn, repo_config = load_tools_config(self.local_repo_dir)
@@ -326,6 +331,21 @@ def component_files_identical(self, component_name, base_path, commit, component
         self.checkout_branch()
         return files_identical
 
+    def ensure_git_user_config(self, default_name: str, default_email: str) -> None:
+        try:
+            with self.repo.config_reader() as git_config:
+                user_name = git_config.get_value("user", "name", default=None)
+                user_email = git_config.get_value("user", "email", default=None)
+        except (NoOptionError, NoSectionError):
+            user_name = user_email = None
+
+        if not user_name or not user_email:
+            with self.repo.config_writer() as git_config:
+                if not user_name:
+                    git_config.set_value("user", "name", default_name)
+                if not user_email:
+                    git_config.set_value("user", "email", default_email)
+
     def get_component_git_log(self, component_name, component_type, depth=None):
         """
         Fetches the commit history the of requested module/subworkflow since a given date. The default value is

tests/test_cli.py

@@ -168,6 +168,7 @@ def test_cli_download(self, mock_dl):
             "force": None,
             "platform": None,
             "download-configuration": None,
+            "tag": "3.12=testing",
             "container-system": "singularity",
             "container-library": "quay.io",
             "container-cache-utilisation": "copy",
@@ -188,6 +189,7 @@ def test_cli_download(self, mock_dl):
             "force" in params,
             "platform" in params,
             "download-configuration" in params,
+            (params["tag"],),
             params["container-system"],
             (params["container-library"],),
             params["container-cache-utilisation"],