diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba8eee7b..e5c439279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## [7.32.0](https://github.com/snakemake/snakemake/compare/v7.31.1...v7.32.0) (2023-08-03) + + +### Features + +* add support for Kubernetes service account name spec ([#2254](https://github.com/snakemake/snakemake/issues/2254)) ([3370426](https://github.com/snakemake/snakemake/commit/3370426da7ee78af5de54689f623e2b5afa45f1f)) + + +### Bug Fixes + +* Enable values with an = sign in default_resources ([#2340](https://github.com/snakemake/snakemake/issues/2340)) ([c1c9229](https://github.com/snakemake/snakemake/commit/c1c922904f09c133e39872346a541e7cd216d0d2)) +* Escape workdir paths for potential spaces in paths ([#2196](https://github.com/snakemake/snakemake/issues/2196)) ([9261f7e](https://github.com/snakemake/snakemake/commit/9261f7ea50a8ae424c015faca73b7811fb51d093)) +* ga4gh executor resources ([#2042](https://github.com/snakemake/snakemake/issues/2042)) ([ad6eaef](https://github.com/snakemake/snakemake/commit/ad6eaef6bac05d4de682f59d9d4a088f143b5798)) +* print exceptions when job is not a shell job ([#2385](https://github.com/snakemake/snakemake/issues/2385)) ([8a37b85](https://github.com/snakemake/snakemake/commit/8a37b8584f216ada10caffcbb8b731efd675376a)) +* remote-azblob-sasToken-Authorization ([#1800](https://github.com/snakemake/snakemake/issues/1800)) ([bc854a7](https://github.com/snakemake/snakemake/commit/bc854a7e012cac751b708df83378fd5791e6e6fc)) +* wms-monitor now gets data in correct json format ([#2347](https://github.com/snakemake/snakemake/issues/2347)) ([7fafa7a](https://github.com/snakemake/snakemake/commit/7fafa7ace72f8a727457f4abe6db2f9ed2d74d64)) + + +### Documentation + +* fix a copy&paste (?) mistake ([#2386](https://github.com/snakemake/snakemake/issues/2386)) ([d878847](https://github.com/snakemake/snakemake/commit/d87884749fd9450062f6fde5b7727867396e7a78)) + +## [7.31.1](https://github.com/snakemake/snakemake/compare/v7.31.0...v7.31.1) (2023-08-02) + + +### Bug Fixes + +* require python >=3.7 again (the python 3.9 dependency was unnecessary) ([#2372](https://github.com/snakemake/snakemake/issues/2372)) ([0d0e9c4](https://github.com/snakemake/snakemake/commit/0d0e9c4cf48a97952464e6da476ed7661d629ce3)) + + +### Documentation + +* update CHANGELOG.md: add minimum Python version bump ([#2370](https://github.com/snakemake/snakemake/issues/2370)) ([48e934d](https://github.com/snakemake/snakemake/commit/48e934dcf96e4e8fd30c81cab3674583bf049a45)) + ## [7.31.0](https://github.com/snakemake/snakemake/compare/v7.30.2...v7.31.0) (2023-07-26) @@ -9,6 +43,9 @@ ## [7.30.2](https://github.com/snakemake/snakemake/compare/v7.30.1...v7.30.2) (2023-07-20) +### Breaking changes + +* Bump minimum Python version from 3.7 to 3.9 ([#2369](https://github.com/snakemake/snakemake/issues/2369)) ([4608163](https://github.com/snakemake/snakemake/pull/2341/commits/4608163727bb32e216f1a26adc61d4c15d4b6a47)) ### Bug Fixes diff --git a/docs/snakefiles/rules.rst b/docs/snakefiles/rules.rst index 44445aee1..c8c8589ef 100644 --- a/docs/snakefiles/rules.rst +++ b/docs/snakefiles/rules.rst @@ -321,7 +321,7 @@ To quickly exemplify the latter, you could provide the following workflow profil set-threads: b: 16 -to set the memory requirement of rule ``b`` to 1000 MB. +to set the (maximum) number of threads rule ``b`` uses to 16. .. _snakefiles-resources: diff --git a/docs/tutorial/basics.rst b/docs/tutorial/basics.rst index 5b29068be..a2b1bc779 100644 --- a/docs/tutorial/basics.rst +++ b/docs/tutorial/basics.rst @@ -14,7 +14,9 @@ Basics: An example workflow .. _Miniconda: https://conda.pydata.org/miniconda.html .. _Conda: https://conda.pydata.org .. _Bash: https://www.tldp.org/LDP/Bash-Beginners-Guide/html -.. _Atom: https://atom.io +.. _Visual Studio Code: https://code.visualstudio.com/ +.. _Snakemake extension: https://marketplace.visualstudio.com/items?itemName=Snakemake.snakemake-lang +.. _remote extension: https://marketplace.visualstudio.com/items?itemName=ms-vscode.remote-explorer .. _Anaconda: https://anaconda.org .. _Graphviz: https://www.graphviz.org .. _RestructuredText: https://docutils.sourceforge.io/docs/user/rst/quickstart.html @@ -83,7 +85,7 @@ Step 1: Mapping reads Our first Snakemake rule maps reads of a given sample to a given reference genome (see :ref:`tutorial-background`). For this, we will use the tool bwa_, specifically the subcommand ``bwa mem``. In the working directory, **create a new file** called ``Snakefile`` with an editor of your choice. -We propose to use the Atom_ editor, since it provides out-of-the-box syntax highlighting for Snakemake. +We propose to use the integrated development environment (IDE) tool `Visual Studio Code`_, since it provides a good syntax highlighting `Snakemake extension`_ and a `remote extension`_ for directly using the IDE on a remote server. In the Snakefile, define the following rule: .. code:: python diff --git a/setup.cfg b/setup.cfg index 62bdfc876..5fa20aa34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ classifiers = zip_safe = False include_package_data = False packages = find: -python_requires = >=3.9 +python_requires = >=3.7 install_requires = appdirs configargparse diff --git a/snakemake/api.py b/snakemake/api.py index c10088072..334023b1b 100644 --- a/snakemake/api.py +++ b/snakemake/api.py @@ -152,6 +152,7 @@ def snakemake( kubernetes=None, container_image=None, k8s_cpu_scalar=1.0, + k8s_service_account_name=None, flux=False, tibanna=False, tibanna_sfn=None, @@ -295,6 +296,7 @@ def snakemake( kubernetes (str): submit jobs to Kubernetes, using the given namespace. container_image (str): Docker image to use, e.g., for Kubernetes. k8s_cpu_scalar (float): What proportion of each k8s node's CPUs are availabe to snakemake? + k8s_service_account_name (str): Custom k8s service account, needed for workload identity. flux (bool): Launch workflow to flux cluster. default_remote_provider (str): default remote provider to use instead of local files (e.g. S3, GS) default_remote_prefix (str): prefix for default remote provider (e.g. name of the bucket). @@ -706,6 +708,7 @@ def snakemake( kubernetes=kubernetes, container_image=container_image, k8s_cpu_scalar=k8s_cpu_scalar, + k8s_service_account_name=k8s_service_account_name, conda_create_envs_only=conda_create_envs_only, default_remote_provider=default_remote_provider, default_remote_prefix=default_remote_prefix, @@ -769,6 +772,7 @@ def snakemake( kubernetes=kubernetes, container_image=container_image, k8s_cpu_scalar=k8s_cpu_scalar, + k8s_service_account_name=k8s_service_account_name, tibanna=tibanna, tibanna_sfn=tibanna_sfn, az_batch=az_batch, diff --git a/snakemake/cli.py b/snakemake/cli.py index a3ff3422a..1818d94ca 100644 --- a/snakemake/cli.py +++ b/snakemake/cli.py @@ -1609,6 +1609,16 @@ def get_argument_parser(profiles=None): "see the original value, i.e. as the value substituted in {threads}.", ) + group_kubernetes.add_argument( + "--k8s-service-account-name", + metavar="SERVICEACCOUNTNAME", + default=None, + help="This argument allows the use of customer service accounts for " + "kubernetes pods. If specified serviceAccountName will be added to the " + "pod specs. This is needed when using workload identity which is enforced " + "when using Google Cloud GKE Autopilot.", + ) + group_tibanna.add_argument( "--tibanna", action="store_true", @@ -2322,6 +2332,7 @@ def open_browser(): kubernetes=args.kubernetes, container_image=args.container_image, k8s_cpu_scalar=args.k8s_cpu_scalar, + k8s_service_account_name=args.k8s_service_account_name, flux=args.flux, tibanna=args.tibanna, tibanna_sfn=args.tibanna_sfn, diff --git a/snakemake/executors/__init__.py b/snakemake/executors/__init__.py index 5d7591a6a..35ee91104 100644 --- a/snakemake/executors/__init__.py +++ b/snakemake/executors/__init__.py @@ -20,6 +20,7 @@ from functools import partial from collections import namedtuple import base64 +from typing import List import uuid import re import math @@ -191,7 +192,7 @@ def job_specific_local_groupid(self): return False def get_job_exec_prefix(self, job: ExecutorJobInterface): - return f"cd {self.workflow.workdir_init}" + return f"cd {shlex.quote(self.workflow.workdir_init)}" def get_python_executable(self): return sys.executable @@ -366,7 +367,7 @@ def _callback( error_callback(job) except (Exception, BaseException) as ex: self.print_job_error(job) - if self.workflow.verbose or (not job.is_group() and job.is_run): + if self.workflow.verbose or (not job.is_group() and not job.is_shell): print_exception(ex, self.workflow.linemaps) error_callback(job) @@ -439,8 +440,7 @@ def __init__( def get_job_exec_prefix(self, job: ExecutorJobInterface): if self.assume_shared_fs: - # quoting the workdir since it may contain spaces - return f"cd {repr(self.workflow.workdir_init)}" + return f"cd {shlex.quote(self.workflow.workdir_init)}" else: return "" @@ -785,7 +785,7 @@ def __init__( def get_job_exec_prefix(self, job): if self.assume_shared_fs: - return f"cd {self.workflow.workdir_init}" + return f"cd {shlex.quote(self.workflow.workdir_init)}" else: return "" @@ -902,7 +902,7 @@ def __init__( def get_job_exec_prefix(self, job: ExecutorJobInterface): if self.assume_shared_fs: - return f"cd {self.workflow.workdir_init}" + return f"cd {shlex.quote(self.workflow.workdir_init)}" else: return "" @@ -1069,6 +1069,7 @@ def __init__( namespace, container_image=None, k8s_cpu_scalar=1.0, + k8s_service_account_name=None, jobname="{rulename}.{jobid}", ): self.workflow = workflow @@ -1098,6 +1099,7 @@ def __init__( import kubernetes.client self.k8s_cpu_scalar = k8s_cpu_scalar + self.k8s_service_account_name = k8s_service_account_name self.kubeapi = kubernetes.client.CoreV1Api() self.batchapi = kubernetes.client.BatchV1Api() self.namespace = namespace @@ -1284,6 +1286,10 @@ def run( body.spec = kubernetes.client.V1PodSpec( containers=[container], node_selector=node_selector ) + # Add service account name if provided + if self.k8s_service_account_name: + body.spec.service_account_name = self.k8s_service_account_name + # fail on first error body.spec.restart_policy = "Never" diff --git a/snakemake/executors/ga4gh_tes.py b/snakemake/executors/ga4gh_tes.py index 57aeec273..5588f303f 100644 --- a/snakemake/executors/ga4gh_tes.py +++ b/snakemake/executors/ga4gh_tes.py @@ -4,6 +4,7 @@ __license__ = "MIT" import asyncio +import math import os from collections import namedtuple @@ -310,12 +311,12 @@ def _get_task(self, job: ExecutorJobInterface, jobscript): task["resources"] = tes.models.Resources() # define resources - if "_cores" in job.resources: - task["resources"]["cpu_cores"] = job.resources["_cores"] - if "mem_mb" in job.resources: - task["resources"]["ram_gb"] = job.resources["mem_mb"] / 1000 - if "disk_mb" in job.resources: - task["resources"]["disk_gb"] = job.resources["disk_mb"] / 1000 + if job.resources.get("_cores") is not None: + task["resources"].cpu_cores = job.resources["_cores"] + if job.resources.get("mem_mb") is not None: + task["resources"].ram_gb = math.ceil(job.resources["mem_mb"] / 1000) + if job.resources.get("disk_mb") is not None: + task["resources"].disk_gb = math.ceil(job.resources["disk_mb"] / 1000) tes_task = tes.Task(**task) logger.debug(f"[TES] Built task: {tes_task}") diff --git a/snakemake/logging.py b/snakemake/logging.py index 3286de572..6617a7099 100644 --- a/snakemake/logging.py +++ b/snakemake/logging.py @@ -185,7 +185,7 @@ def create_workflow(self): f"{self.address}/create_workflow", headers=self._headers, params=self.args, - data=metadata, + data=json.dumps(metadata), ) # Check the response, will exit on any error @@ -276,7 +276,9 @@ def log_handler(self, msg): "timestamp": time.asctime(), "id": self.server["id"], } - response = requests.post(url, data=server_info, headers=self._headers) + response = requests.post( + url, data=json.dumps(server_info), headers=self._headers + ) self.check_response(response, "/update_workflow_status") diff --git a/snakemake/remote/AzBlob.py b/snakemake/remote/AzBlob.py index 2fbd77100..280359124 100644 --- a/snakemake/remote/AzBlob.py +++ b/snakemake/remote/AzBlob.py @@ -244,10 +244,17 @@ def upload_to_azure_storage( ) container_client = self.blob_service_client.get_container_client(container_name) + + # create container if it doesn't exist. + # for sas token created in the level of container, the exists method will fail with error code 403. + # therefore the exception is passed to cover this type of sas tokens. + try: - container_client.create_container() - except azure.core.exceptions.ResourceExistsError: - pass + if not container_client.exists(): + container_client.create_container() + except Exception as e: + if e.status_code == 403: + pass if not blob_name: if use_relative_path_for_blob_name: diff --git a/snakemake/resources.py b/snakemake/resources.py index a343cf458..3dd052e64 100644 --- a/snakemake/resources.py +++ b/snakemake/resources.py @@ -20,7 +20,7 @@ class DefaultResources: @classmethod def decode_arg(cls, arg): try: - return arg.split("=") + return arg.split("=", maxsplit=1) except ValueError: raise ValueError("Resources have to be defined as name=value pairs.") diff --git a/snakemake/scheduler.py b/snakemake/scheduler.py index d1b38200f..7526a97ac 100644 --- a/snakemake/scheduler.py +++ b/snakemake/scheduler.py @@ -85,6 +85,7 @@ def __init__( env_modules=None, kubernetes=None, k8s_cpu_scalar=1.0, + k8s_service_account_name=None, container_image=None, flux=None, tibanna=None, @@ -312,6 +313,7 @@ def __init__( kubernetes, container_image=container_image, k8s_cpu_scalar=k8s_cpu_scalar, + k8s_service_account_name=k8s_service_account_name, ) elif tibanna: self._local_executor = CPUExecutor( diff --git a/snakemake/workflow.py b/snakemake/workflow.py index e6dfbd372..485e4e8e2 100644 --- a/snakemake/workflow.py +++ b/snakemake/workflow.py @@ -691,6 +691,7 @@ def execute( drmaa_log_dir=None, kubernetes=None, k8s_cpu_scalar=1.0, + k8s_service_account_name=None, flux=None, tibanna=None, tibanna_sfn=None, @@ -1144,6 +1145,7 @@ def files(items): drmaa_log_dir=drmaa_log_dir, kubernetes=kubernetes, k8s_cpu_scalar=k8s_cpu_scalar, + k8s_service_account_name=k8s_service_account_name, flux=flux, tibanna=tibanna, tibanna_sfn=tibanna_sfn, diff --git a/test-environment.yml b/test-environment.yml index 20ce61ed6..cfc1729c7 100644 --- a/test-environment.yml +++ b/test-environment.yml @@ -2,7 +2,7 @@ channels: - conda-forge - bioconda dependencies: - - python >=3.9 + - python >=3.7 - yte - packaging - stopit diff --git a/tests/test_conda_python_3_7_script/Snakefile b/tests/test_conda_python_3_7_script/Snakefile new file mode 100644 index 000000000..09ecae61f --- /dev/null +++ b/tests/test_conda_python_3_7_script/Snakefile @@ -0,0 +1,7 @@ +rule random_python_conda_script: + output: + "version.txt" + conda: + "test_python_env.yaml" + script: + "test_script.py" diff --git a/tests/test_conda_python_3_7_script/expected-results/version.txt b/tests/test_conda_python_3_7_script/expected-results/version.txt new file mode 100644 index 000000000..b44c53705 --- /dev/null +++ b/tests/test_conda_python_3_7_script/expected-results/version.txt @@ -0,0 +1 @@ +3.7.12 \ No newline at end of file diff --git a/tests/test_conda_python_3_7_script/test_python_env.yaml b/tests/test_conda_python_3_7_script/test_python_env.yaml new file mode 100644 index 000000000..952461ac3 --- /dev/null +++ b/tests/test_conda_python_3_7_script/test_python_env.yaml @@ -0,0 +1,9 @@ +channels: + - conda-forge + - defaults +dependencies: + - python =3.7.12 + # add a dependency that is not used by Snakemake itself, + # to simulate the case where the Python script needs 3.7 and + # that dependency + - pillow =9.2 diff --git a/tests/test_conda_python_3_7_script/test_script.py b/tests/test_conda_python_3_7_script/test_script.py new file mode 100644 index 000000000..8939f5e3c --- /dev/null +++ b/tests/test_conda_python_3_7_script/test_script.py @@ -0,0 +1,5 @@ +import platform +import PIL + +with open('version.txt', 'w') as f: + f.write(platform.python_version()) diff --git a/tests/test_slurm.py b/tests/test_slurm.py index 88e6822e0..0f49348e1 100644 --- a/tests/test_slurm.py +++ b/tests/test_slurm.py @@ -80,3 +80,20 @@ def test_slurm_complex(): ] ), ) + + +@skip_on_windows +def test_slurm_extra_arguments(): + """Make sure arguments to default resources + are allowed to contain = signs, which is needed + for extra slurm arguments""" + run( + dpath("test_slurm_mpi"), + slurm=True, + show_failed_logs=True, + use_conda=True, + default_resources=DefaultResources( + ["slurm_account=runner", "slurm_partition=debug", + "slurm_extra='--mail-type=none'"] + ), + ) diff --git a/tests/tests.py b/tests/tests.py index 36296a5b7..6a555bd6d 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -2023,6 +2023,10 @@ def test_conda_python_script(): run(dpath("test_conda_python_script"), use_conda=True) +def test_conda_python_3_7_script(): + run(dpath("test_conda_python_3_7_script"), use_conda=True) + + def test_prebuilt_conda_script(): sp.run("conda env create -f tests/test_prebuilt_conda_script/env.yaml", shell=True) run(dpath("test_prebuilt_conda_script"), use_conda=True)