diff --git a/CHANGELOG.md b/CHANGELOG.md index 88ce77a3b..d6218f7d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v2.13.2dev +## v2.14.0dev ### Template @@ -25,8 +25,9 @@ ### Download -- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. +- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. ([#2853](https://github.com/nf-core/tools/pull/2853)) - Better error message when GITHUB_TOKEN exists but is wrong/outdated +- New `--tag` argument to add custom tags during a pipeline download ([#2938](https://github.com/nf-core/tools/pull/2938)) ### Components diff --git a/README.md b/README.md index a5e679986..522285408 100644 --- a/README.md +++ b/README.md @@ -437,6 +437,9 @@ Subsequently, the `*.git` folder can be moved to it's final destination and link > [!TIP] > Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. +Facilities and those who are setting up pipelines for others to use may find the `--tag` argument helpful. It allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `--tag "3.12.0=testing" --tag "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. +The `--tag` argument must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. + ## Pipeline software licences Sometimes it's useful to see the software licences of the tools used in a pipeline. diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 807bc776b..39a1afbf6 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -368,14 +368,14 @@ def create_params_file(pipeline, revision, output, force, show_hidden): @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") # TODO: Remove this in a future release. Deprecated in March 2024. @click.option( + "-t", "--tower", is_flag=True, default=False, hidden=True, - help="Download for Seqera Platform. DEPRECATED: Please use --platform instead.", + help="Download for Seqera Platform. DEPRECATED: Please use `--platform` instead.", ) @click.option( - "-t", "--platform", is_flag=True, default=False, @@ -388,6 +388,11 @@ def create_params_file(pipeline, revision, output, force, show_hidden): default=False, help="Include configuration profiles in download. Not available with `--platform`", ) +@click.option( + "--tag", + multiple=True, + help="Add custom alias tags to `--platform` downloads. For example, `--tag \"3.10=validated\"` adds the custom 'validated' tag to the 3.10 release.", +) # -c changed to -s for consistency with other --container arguments, where it is always the first letter of the last word. # Also -c might be used instead of -d for config in a later release, but reusing params for different options in two subsequent releases might be too error-prone. @click.option( @@ -430,6 +435,7 @@ def download( tower, platform, download_configuration, + tag, container_system, container_library, container_cache_utilisation, @@ -444,6 +450,9 @@ def download( """ from nf_core.download import DownloadWorkflow + if tower: + log.warning("[red]The `-t` / `--tower` flag is deprecated. Please use `--platform` instead.[/]") + dl = DownloadWorkflow( pipeline, revision, @@ -452,6 +461,7 @@ def download( force, tower or platform, # True if either specified download_configuration, + tag, container_system, container_library, container_cache_utilisation, diff --git a/nf_core/download.py b/nf_core/download.py index 8ffecc0f5..f5ab3a0f5 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -88,10 +88,18 @@ class DownloadWorkflow: Args: pipeline (str): A nf-core pipeline name. - revision (List[str]): The workflow revision to download, like `1.0`. Defaults to None. - container (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. - platform (bool): Flag, to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. + revision (List[str]): The workflow revision(s) to download, like `1.0` or `dev` . Defaults to None. outdir (str): Path to the local download directory. Defaults to None. + compress_type (str): Type of compression for the downloaded files. Defaults to None. + force (bool): Flag to force download even if files already exist (overwrite existing files). Defaults to False. + platform (bool): Flag to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. + download_configuration (str): Download the configuration files from nf-core/configs. Defaults to None. + tag (List[str]): Specify additional tags to add to the downloaded pipeline. Defaults to None. + container_system (str): The container system to use (e.g., "singularity"). Defaults to None. + container_library (List[str]): The container libraries (registries) to use. Defaults to None. + container_cache_utilisation (str): If a local or remote cache of already existing container images should be considered. Defaults to None. + container_cache_index (str): An index for the remote container cache. Defaults to None. + parallel_downloads (int): The number of parallel downloads to use. Defaults to 4. """ def __init__( @@ -103,6 +111,7 @@ def __init__( force=False, platform=False, download_configuration=None, + additional_tags=None, container_system=None, container_library=None, container_cache_utilisation=None, @@ -125,6 +134,15 @@ def __init__( # this implies that non-interactive "no" choice is only possible implicitly (e.g. with --platform or if prompt is suppressed by !stderr.is_interactive). # only alternative would have been to make it a parameter with argument, e.g. -d="yes" or -d="no". self.include_configs = True if download_configuration else False if bool(platform) else None + # Additional tags to add to the downloaded pipeline. This enables to mark particular commits or revisions with + # additional tags, e.g. "stable", "testing", "validated", "production" etc. Since this requires a git-repo, it is only + # available for the bare / Seqera Platform download. + if isinstance(additional_tags, str) and bool(len(additional_tags)) and self.platform: + self.additional_tags = [additional_tags] + elif isinstance(additional_tags, tuple) and bool(len(additional_tags)) and self.platform: + self.additional_tags = [*additional_tags] + else: + self.additional_tags = None # Specifying a cache index or container library implies that containers should be downloaded. self.container_system = "singularity" if container_cache_index or bool(container_library) else container_system # Manually specified container library (registry) @@ -282,6 +300,7 @@ def download_workflow_platform(self, location=None): remote_url=f"https://github.com/{self.pipeline}.git", revision=self.revision if self.revision else None, commit=self.wf_sha.values() if bool(self.wf_sha) else None, + additional_tags=self.additional_tags, location=(location if location else None), # manual location is required for the tests to work in_cache=False, ) @@ -1489,6 +1508,7 @@ def __init__( remote_url, revision, commit, + additional_tags, location=None, hide_progress=False, in_cache=True, @@ -1523,13 +1543,21 @@ def __init__( self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache) - # expose some instance attributes - self.tags = self.repo.tags + # additional tags to be added to the repository + self.additional_tags = additional_tags if additional_tags else None def __repr__(self): """Called by print, creates representation of object""" return f"" + @property + def heads(self): + return self.repo.heads + + @property + def tags(self): + return self.repo.tags + def access(self): if os.path.exists(self.local_repo_dir): return self.local_repo_dir @@ -1640,7 +1668,6 @@ def tidy_tags_and_branches(self): # delete unwanted tags from repository for tag in tags_to_remove: self.repo.delete_tag(tag) - self.tags = self.repo.tags # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master") self.checkout(self.revision[0]) @@ -1677,7 +1704,8 @@ def tidy_tags_and_branches(self): if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) - self.heads = self.repo.heads + # Apply the custom additional tags to the repository + self.__add_additional_tags() # get all tags and available remote_branches completed_revisions = {revision.name for revision in self.repo.heads + self.repo.tags} @@ -1695,6 +1723,39 @@ def tidy_tags_and_branches(self): self.retry_setup_local_repo(skip_confirm=True) raise DownloadError(e) from e + # "Private" method to add the additional custom tags to the repository. + def __add_additional_tags(self) -> None: + if self.additional_tags: + # example.com is reserved by the Internet Assigned Numbers Authority (IANA) as special-use domain names for documentation purposes. + # Although "dev-null" is a syntactically-valid local-part that is equally valid for delivery, + # and only the receiving MTA can decide whether to accept it, it is to my best knowledge configured with + # a Postfix discard mail delivery agent (https://www.postfix.org/discard.8.html), so incoming mails should be sinkholed. + self.ensure_git_user_config(f"nf-core download v{nf_core.__version__}", "dev-null@example.com") + + for additional_tag in self.additional_tags: + # A valid git branch or tag name can contain alphanumeric characters, underscores, hyphens, and dots. + # But it must not start with a dot, hyphen or underscore and also cannot contain two consecutive dots. + if re.match(r"^\w[\w_.-]+={1}\w[\w_.-]+$", additional_tag) and ".." not in additional_tag: + anchor, tag = additional_tag.split("=") + if self.repo.is_valid_object(anchor) and not self.repo.is_valid_object(tag): + try: + self.repo.create_tag( + tag, ref=anchor, message=f"Synonynmous tag to {anchor}; added by `nf-core download`." + ) + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n") + else: + if not self.repo.is_valid_object(anchor): + log.error( + f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit." + ) + if self.repo.is_valid_object(tag): + log.error( + f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{tag}' must not exist hitherto." + ) + else: + log.error(f"[red]Could not apply invalid `--tag` specification[/]: '{additional_tag}'") + def bare_clone(self, destination): if self.repo: try: diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 5c31e9691..d4f302254 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -2,6 +2,7 @@ import logging import os import shutil +from configparser import NoOptionError, NoSectionError from pathlib import Path from typing import Dict @@ -116,6 +117,10 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url + self.repo = None + # TODO: SyncedRepo doesn't have this method and both the ModulesRepo and + # the WorkflowRepo define their own including custom init methods. This needs + # fixing. self.setup_local_repo(remote_url, branch, hide_progress) config_fn, repo_config = load_tools_config(self.local_repo_dir) @@ -326,6 +331,21 @@ def component_files_identical(self, component_name, base_path, commit, component self.checkout_branch() return files_identical + def ensure_git_user_config(self, default_name: str, default_email: str) -> None: + try: + with self.repo.config_reader() as git_config: + user_name = git_config.get_value("user", "name", default=None) + user_email = git_config.get_value("user", "email", default=None) + except (NoOptionError, NoSectionError): + user_name = user_email = None + + if not user_name or not user_email: + with self.repo.config_writer() as git_config: + if not user_name: + git_config.set_value("user", "name", default_name) + if not user_email: + git_config.set_value("user", "email", default_email) + def get_component_git_log(self, component_name, component_type, depth=None): """ Fetches the commit history the of requested module/subworkflow since a given date. The default value is diff --git a/tests/test_cli.py b/tests/test_cli.py index 913a4aac1..5df0a3241 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -168,6 +168,7 @@ def test_cli_download(self, mock_dl): "force": None, "platform": None, "download-configuration": None, + "tag": "3.12=testing", "container-system": "singularity", "container-library": "quay.io", "container-cache-utilisation": "copy", @@ -188,6 +189,7 @@ def test_cli_download(self, mock_dl): "force" in params, "platform" in params, "download-configuration" in params, + (params["tag"],), params["container-system"], (params["container-library"],), params["container-cache-utilisation"], diff --git a/tests/test_download.py b/tests/test_download.py index 14a96be26..3e0f11d57 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,11 +1,13 @@ """Tests for the download subcommand of nf-core tools""" +import logging import os import re import shutil import tempfile import unittest from pathlib import Path +from typing import List from unittest import mock import pytest @@ -20,6 +22,24 @@ class DownloadTest(unittest.TestCase): + @pytest.fixture(autouse=True) + def use_caplog(self, caplog): + self._caplog = caplog + + @property + def logged_levels(self) -> List[str]: + return [record.levelname for record in self._caplog.records] + + @property + def logged_messages(self) -> List[str]: + return [record.message for record in self._caplog.records] + + def __contains__(self, item: str) -> bool: + """Allows to check for log messages easily using the in operator inside a test: + assert 'my log message' in self + """ + return any(record.message == item for record in self._caplog.records if self._caplog) + # # Tests for 'get_release_hash' # @@ -623,3 +643,88 @@ def test_download_workflow_for_platform(self, tmp_dir, _): "https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:59cdd445419f14abac76b31dd0d71217994cbcc9-0" in download_obj.containers ) # indirect definition via $container variable. + + # + # Brief test adding a single custom tag to Seqera Platform download + # + @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") + @with_temporary_folder + def test_download_workflow_for_platform_with_one_custom_tag(self, _, tmp_dir): + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=("3.9=cool_revision",), + ) + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 1 + + # + # Test adding custom tags to Seqera Platform download (full test) + # + @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") + @with_temporary_folder + def test_download_workflow_for_platform_with_custom_tags(self, _, tmp_dir): + with self._caplog.at_level(logging.INFO): + from git.refs.tag import TagReference + + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.7", "3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=( + "3.7=a.tad.outdated", + "3.9=cool_revision", + "3.9=invalid tag", + "3.14.0=not_included", + "What is this?", + ), + ) + + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + + assert isinstance(download_obj.revision, list) and len(download_obj.revision) == 2 + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 0 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 5 + + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) + + download_obj.get_revision_hash() + download_obj.output_filename = f"{download_obj.outdir}.git" + download_obj.download_workflow_platform(location=tmp_dir) + + assert download_obj.workflow_repo + assert isinstance(download_obj.workflow_repo, WorkflowRepo) + assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + assert "Locally cached repository: nf-core/rnaseq, revisions 3.7, 3.9" in repr(download_obj.workflow_repo) + + # assert that every additional tag has been passed on to the WorkflowRepo instance + assert download_obj.additional_tags == download_obj.workflow_repo.additional_tags + + # assert that the additional tags are all TagReference objects + assert all(isinstance(tag, TagReference) for tag in download_obj.workflow_repo.tags) + + workflow_repo_tags = {tag.name for tag in download_obj.workflow_repo.tags} + assert len(workflow_repo_tags) == 4 + # the invalid/malformed additional_tags should not have been added. + assert all(tag in workflow_repo_tags for tag in {"3.7", "a.tad.outdated", "cool_revision", "3.9"}) + assert not any(tag in workflow_repo_tags for tag in {"invalid tag", "not_included", "What is this?"}) + + assert all( + log in self.logged_messages + for log in { + "[red]Could not apply invalid `--tag` specification[/]: '3.9=invalid tag'", + "[red]Adding tag 'not_included' to '3.14.0' failed.[/]\n Mind that '3.14.0' must be a valid git reference that resolves to a commit.", + "[red]Could not apply invalid `--tag` specification[/]: 'What is this?'", + } + )