diff --git a/cwl_utils/cwl_v1_0_expression_refactor.py b/cwl_utils/cwl_v1_0_expression_refactor.py index 5b5a5e92..c47d323a 100755 --- a/cwl_utils/cwl_v1_0_expression_refactor.py +++ b/cwl_utils/cwl_v1_0_expression_refactor.py @@ -6,6 +6,7 @@ import hashlib import uuid from collections.abc import Mapping, MutableSequence, Sequence +from contextlib import suppress from typing import Any, Optional, cast from ruamel import yaml @@ -30,11 +31,9 @@ def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool result = copy.deepcopy(process) stdout_path = process.stdout if not stdout_path: - stdout_path = str( - hashlib.sha1( # nosec - json_dumps(cwl.save(process)).encode("utf-8") - ).hexdigest() - ) + stdout_path = hashlib.sha1( # nosec + json_dumps(cwl.save(process)).encode("utf-8") + ).hexdigest() result.stdout = stdout_path result.outputs[index].type_ = "File" output.outputBinding = cwl.CommandOutputBinding(stdout_path, None, None) @@ -534,12 +533,10 @@ def empty_inputs( elif param.source is None and param.default: result[param_id] = param.default else: - try: + with suppress(WorkflowException): result[param_id] = example_input( utils.type_for_source(process_or_step.run, param.source, parent) ) - except WorkflowException: - pass return result @@ -797,8 +794,8 @@ def process_workflow_reqs_and_hints( isinstance(expr_result, Mapping) and "class" in expr_result and ( - expr_result["class"] == "File" - or expr_result["class"] == "Directory" + expr_result["class"] + in ("File", "Directory") ) ): target = cwl.InputParameter( diff --git a/cwl_utils/cwl_v1_1_expression_refactor.py b/cwl_utils/cwl_v1_1_expression_refactor.py index 286862af..63fe6f81 100755 --- a/cwl_utils/cwl_v1_1_expression_refactor.py +++ b/cwl_utils/cwl_v1_1_expression_refactor.py @@ -6,6 +6,7 @@ import hashlib import uuid from collections.abc import Mapping, MutableSequence, Sequence +from contextlib import suppress from typing import Any, Optional, cast from ruamel import yaml @@ -30,11 +31,9 @@ def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool result = copy.deepcopy(process) stdout_path = process.stdout if not stdout_path: - stdout_path = str( - hashlib.sha1( # nosec - json_dumps(cwl.save(process)).encode("utf-8") - ).hexdigest() - ) + stdout_path = hashlib.sha1( # nosec + json_dumps(cwl.save(process)).encode("utf-8") + ).hexdigest() result.stdout = stdout_path result.outputs[index].type_ = "File" output.outputBinding = cwl.CommandOutputBinding(stdout_path, None, None) @@ -532,12 +531,10 @@ def empty_inputs( elif param.source is None and param.default: result[param_id] = param.default else: - try: + with suppress(WorkflowException): result[param_id] = example_input( utils.type_for_source(process_or_step.run, param.source, parent) ) - except WorkflowException: - pass return result @@ -799,8 +796,8 @@ def process_workflow_reqs_and_hints( isinstance(expr_result, Mapping) and "class" in expr_result and ( - expr_result["class"] == "File" - or expr_result["class"] == "Directory" + expr_result["class"] + in ("File", "Directory") ) ): target = cwl.WorkflowInputParameter( diff --git a/cwl_utils/cwl_v1_2_expression_refactor.py b/cwl_utils/cwl_v1_2_expression_refactor.py index 48c8c706..a56c7950 100755 --- a/cwl_utils/cwl_v1_2_expression_refactor.py +++ b/cwl_utils/cwl_v1_2_expression_refactor.py @@ -6,6 +6,7 @@ import hashlib import uuid from collections.abc import Mapping, MutableSequence, Sequence +from contextlib import suppress from typing import Any, Optional, cast from ruamel import yaml @@ -30,11 +31,9 @@ def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool result = copy.deepcopy(process) stdout_path = process.stdout if not stdout_path: - stdout_path = str( - hashlib.sha1( # nosec - json_dumps(cwl.save(process)).encode("utf-8") - ).hexdigest() - ) + stdout_path = hashlib.sha1( # nosec + json_dumps(cwl.save(process)).encode("utf-8") + ).hexdigest() result.stdout = stdout_path result.outputs[index].type_ = "File" output.outputBinding = cwl.CommandOutputBinding(stdout_path, None, None) @@ -532,12 +531,10 @@ def empty_inputs( elif param.source is None and param.default: result[param_id] = param.default else: - try: + with suppress(WorkflowException): result[param_id] = example_input( utils.type_for_source(process_or_step.run, param.source, parent) ) - except WorkflowException: - pass return result @@ -894,8 +891,8 @@ def process_workflow_reqs_and_hints( isinstance(expr_result, Mapping) and "class" in expr_result and ( - expr_result["class"] == "File" - or expr_result["class"] == "Directory" + expr_result["class"] + in ("File", "Directory") ) ): target = cwl.WorkflowInputParameter( diff --git a/cwl_utils/docker_extract.py b/cwl_utils/docker_extract.py index b702fc4b..f67163d0 100755 --- a/cwl_utils/docker_extract.py +++ b/cwl_utils/docker_extract.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 import argparse -import os import sys from collections.abc import Iterator +from pathlib import Path from typing import cast import ruamel.yaml @@ -51,7 +51,7 @@ def run(args: argparse.Namespace) -> list[cwl.DockerRequirement]: sys.exit(1) if args.dir: - os.makedirs(args.dir, exist_ok=True) + Path(args.dir).mkdir(parents=True, exist_ok=True) top = cwl.load_document_by_uri(args.input) reqs: list[cwl.DockerRequirement] = [] diff --git a/cwl_utils/expression.py b/cwl_utils/expression.py index b72d09b4..fe04e87f 100644 --- a/cwl_utils/expression.py +++ b/cwl_utils/expression.py @@ -252,8 +252,7 @@ def dump(string: str) -> str: scan = scan[w[1] :] w = scanner(scan) if convert_to_expression: - parts.append(f'"{scan}"') # noqa: B907 - parts.append(";}") + parts.extend((f'"{scan}"', ";}")) # noqa: B907 else: parts.append(scan) return "".join(parts) @@ -290,8 +289,8 @@ def do_eval( :param timeout: The maximum number of seconds to wait while executing. """ runtime = cast(MutableMapping[str, Union[int, str, None]], copy.deepcopy(resources)) - runtime["tmpdir"] = tmpdir if tmpdir else None - runtime["outdir"] = outdir if outdir else None + runtime["tmpdir"] = tmpdir or None + runtime["outdir"] = outdir or None rootvars = cast( CWLObjectType, diff --git a/cwl_utils/expression_refactor.py b/cwl_utils/expression_refactor.py index eca02ece..9784cd64 100755 --- a/cwl_utils/expression_refactor.py +++ b/cwl_utils/expression_refactor.py @@ -139,11 +139,7 @@ def refactor(args: argparse.Namespace) -> int: if not isinstance(result, MutableSequence): result_json = save( result, - base_url=( - result.loadingOptions.fileuri - if result.loadingOptions.fileuri - else "" - ), + base_url=(result.loadingOptions.fileuri or ""), ) # ^^ Setting the base_url and keeping the default value # for relative_uris=True means that the IDs in the generated @@ -155,7 +151,7 @@ def refactor(args: argparse.Namespace) -> int: ] walk_tree(result_json) # ^ converts multiline strings to nice multiline YAML - with open(output, "w", encoding="utf-8") as output_filehandle: + with output.open("w", encoding="utf-8") as output_filehandle: output_filehandle.write( "#!/usr/bin/env cwl-runner\n" ) # TODO: teach the codegen to do this? diff --git a/cwl_utils/graph_split.py b/cwl_utils/graph_split.py index e2ba20ef..ad78d954 100755 --- a/cwl_utils/graph_split.py +++ b/cwl_utils/graph_split.py @@ -67,7 +67,7 @@ def arg_parser() -> argparse.ArgumentParser: "-C", "--outdir", type=str, - default=os.getcwd(), + default=Path.cwd(), help="Output folder for the unpacked CWL files.", ) return parser @@ -194,7 +194,7 @@ def rewrite( else: document[key] = f"{referrant_file}#{sub}" elif isinstance(value, list): - new_sources = list() + new_sources = [] for entry in value: if entry.startswith("#" + doc_id): new_sources.append(entry[len(doc_id) + 2 :]) diff --git a/cwl_utils/image_puller.py b/cwl_utils/image_puller.py index da5452cc..cb7be35b 100644 --- a/cwl_utils/image_puller.py +++ b/cwl_utils/image_puller.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Classes for docker-extract.""" import logging -import os import subprocess # nosec from abc import ABC, abstractmethod from pathlib import Path @@ -53,11 +52,9 @@ class DockerImagePuller(ImagePuller): def get_image_name(self) -> str: """Get the name of the tarball.""" - name = "".join(self.req.split("/")) + ".tar" # Replace colons with underscores in the name. # See https://github.com/containers/podman/issues/489 - name = name.replace(":", "_") - return name + return ("".join(self.req.split("/")) + ".tar").replace(":", "_") def generate_udocker_loading_command(self) -> str: """Generate the udocker loading command.""" @@ -70,14 +67,14 @@ def save_docker_image(self) -> None: ImagePuller._run_command_pull(cmd_pull) _LOGGER.info(f"Image successfully pulled: {self.req}") if self.save_directory: - dest = os.path.join(self.save_directory, self.get_image_name()) + dest = Path(self.save_directory, self.get_image_name()) if self.save_directory and self.force_pull: - os.remove(dest) + dest.unlink() cmd_save = [ self.cmd, "save", "-o", - dest, + str(dest), self.req, ] subprocess.run(cmd_save, check=True) # nosec @@ -111,10 +108,8 @@ def save_docker_image(self) -> None: save_directory: str | Path if self.save_directory: save_directory = self.save_directory - if ( - os.path.exists(os.path.join(save_directory, self.get_image_name())) - and not self.force_pull - ): + target = Path(save_directory, self.get_image_name()) + if target.exists() and not self.force_pull: _LOGGER.info(f"Already cached {self.req} with Singularity.") return _LOGGER.info(f"Pulling {self.req} with Singularity...") @@ -127,7 +122,7 @@ def save_docker_image(self) -> None: cmd_pull.extend( [ "--name", - os.path.join(save_directory, self.get_image_name()), + str(target), f"docker://{self.req}", ] ) diff --git a/cwl_utils/inputs_schema_gen.py b/cwl_utils/inputs_schema_gen.py index deba943c..e22d067a 100644 --- a/cwl_utils/inputs_schema_gen.py +++ b/cwl_utils/inputs_schema_gen.py @@ -8,6 +8,7 @@ import json import logging import sys +from contextlib import suppress from copy import deepcopy from pathlib import Path from typing import Any, TypeGuard, Union @@ -127,7 +128,7 @@ def generate_type_dict_from_type(self, type_item: Any) -> dict[str, Any]: # Between an CWL Input Parameter type and a JSON schema type match type_item: - case str() as key if key in PRIMITIVE_TYPES_MAPPING.keys(): + case str(key) if key in PRIMITIVE_TYPES_MAPPING: return {"type": PRIMITIVE_TYPES_MAPPING[key]} case "stdin": return {"$ref": "#/definitions/File"} @@ -158,7 +159,7 @@ def generate_type_dict_from_type(self, type_item: Any) -> dict[str, Any]: "type": "string", "enum": list( map( - lambda symbol_iter: get_value_from_uri(symbol_iter), + get_value_from_uri, type_item.symbols, ) ), @@ -171,7 +172,7 @@ def generate_type_dict_from_type(self, type_item: Any) -> dict[str, Any]: match f: case None: return {"type": "object"} - case list() as fields: + case list(fields): return { "type": "object", "properties": { @@ -200,9 +201,7 @@ def generate_type_dict_from_type(self, type_item: Any) -> dict[str, Any]: return { "oneOf": list( map( - lambda type_iter: self.generate_type_dict_from_type( - type_iter - ), + self.generate_type_dict_from_type, type_item, ) ) @@ -217,7 +216,7 @@ def generate_type_dict_from_type_list( return { "oneOf": list( map( - lambda type_item: self.generate_type_dict_from_type(type_item), + self.generate_type_dict_from_type, type_, ) ) @@ -344,7 +343,7 @@ def cwl_to_jsonschema(cwl_obj: Workflow | CommandLineTool) -> Any: """ # Initialise the schema from the workflow input json schema template - with open(JSON_TEMPLATE_PATH) as template_h: + with JSON_TEMPLATE_PATH.open() as template_h: input_json_schema = json.load(template_h) # Get the complex schema keys @@ -367,7 +366,7 @@ def is_complex_record_schema_key(idx_iter: str) -> TypeGuard[bool]: complex_schema_keys: list[str] = list( filter( - lambda idx_iter: is_complex_record_schema_key(idx_iter), + is_complex_record_schema_key, cwl_obj.loadingOptions.idx.keys(), ) ) @@ -389,7 +388,7 @@ def get_complex_schema_values(idx_iter: str) -> InputRecordSchema: complex_schema_values: list[InputRecordSchema] = list( map( - lambda idx_iter: get_complex_schema_values(idx_iter), + get_complex_schema_values, complex_schema_keys, ) ) @@ -397,15 +396,13 @@ def get_complex_schema_values(idx_iter: str) -> InputRecordSchema: # Load in all $imports to be referred by complex input types workflow_schema_definitions_list = list( map( - lambda complex_schema_values_iter: generate_definition_from_schema( - complex_schema_values_iter - ), + generate_definition_from_schema, complex_schema_values, ) ) if cwl_obj.requirements is not None: - try: + with suppress(StopIteration): schema_def_requirement = next( filter( lambda requirement_iter: requirement_iter.class_ @@ -417,17 +414,12 @@ def get_complex_schema_values(idx_iter: str) -> InputRecordSchema: workflow_schema_definitions_list.extend( list( map( - lambda schema_def_iter: generate_definition_from_schema( - schema_def_iter - ), + generate_definition_from_schema, schema_def_requirement.types, ) ) ) - except StopIteration: - pass - # Convert schema definitions to dict workflow_schema_definitions_dict = {} for schema_definition in workflow_schema_definitions_list: @@ -436,9 +428,7 @@ def get_complex_schema_values(idx_iter: str) -> InputRecordSchema: # Generate JSON Schema Properties properties = list( map( - lambda workflow_parameter_input_obj: generate_json_schema_property_from_input_parameter( - workflow_parameter_input_obj - ), + generate_json_schema_property_from_input_parameter, cwl_obj.inputs, ) ) diff --git a/cwl_utils/normalizer.py b/cwl_utils/normalizer.py index 479e6901..c0613e45 100644 --- a/cwl_utils/normalizer.py +++ b/cwl_utils/normalizer.py @@ -100,11 +100,7 @@ def run(args: argparse.Namespace) -> int: if not isinstance(refactored, MutableSequence): result = save( refactored, - base_url=( - refactored.loadingOptions.fileuri - if refactored.loadingOptions.fileuri - else "" - ), + base_url=(refactored.loadingOptions.fileuri or ""), ) # ^^ Setting the base_url and keeping the default value # for relative_uris=True means that the IDs in the generated @@ -121,7 +117,7 @@ def run(args: argparse.Namespace) -> int: path = Path(tmpdirname) / Path(document).name packed = pack(str(path)) output = Path(args.dir) / Path(document).name - with open(output, "w", encoding="utf-8") as output_filehandle: + with output.open("w", encoding="utf-8") as output_filehandle: output_filehandle.write(packed) return 0 diff --git a/cwl_utils/pack.py b/cwl_utils/pack.py index e0c4eb97..0c1bd086 100644 --- a/cwl_utils/pack.py +++ b/cwl_utils/pack.py @@ -35,7 +35,7 @@ def get_inner_dict( cwl: dict[str, Any], path: list[dict[str, Any]] ) -> dict[str, Any] | None: - if len(path) == 0: + if not path: return cwl if isinstance(cwl, dict): @@ -79,7 +79,7 @@ def listify_everything(cwl: dict[str, Any]) -> dict[str, Any]: See https://www.commonwl.org/v1.1/Workflow.html#map """ - for port in ["inputs", "outputs"]: + for port in ("inputs", "outputs"): cwl[port] = utils.normalize_to_list( cwl.get(port, []), key_field="id", value_field="type" ) @@ -137,8 +137,7 @@ def normalize_sources(cwl: dict[str, Any]) -> dict[str, Any]: def _normalize(s: str) -> str: if s.startswith("#"): return s[1:] - else: - return s + return s def load_schemadefs( @@ -182,7 +181,7 @@ def resolve_imports(cwl: Any, base_url: urllib.parse.ParseResult) -> Any: if isinstance(v, dict): if len(v) == 1: _k = list(v.keys())[0] - if _k in ["$import", "$include"]: + if _k in ("$import", "$include"): cwl[k], this_base_url = utils.load_linked_file( base_url, v[_k], is_import=_k == "$import" ) diff --git a/cwl_utils/parser/__init__.py b/cwl_utils/parser/__init__.py index 8d99f5b1..f571085d 100644 --- a/cwl_utils/parser/__init__.py +++ b/cwl_utils/parser/__init__.py @@ -306,7 +306,7 @@ def load_document( ) -> Any: """Load a CWL object from a serialized YAML string or a YAML object.""" if baseuri is None: - baseuri = cwl_v1_0.file_uri(os.getcwd()) + "/" + baseuri = cwl_v1_0.file_uri(str(Path.cwd())) + "/" if isinstance(doc, str): return load_document_by_string(doc, baseuri, loadingOptions, id_) return load_document_by_yaml(doc, baseuri, loadingOptions, id_, load_all) @@ -320,8 +320,7 @@ def load_document_by_string( load_all: bool = False, ) -> Any: """Load a CWL object from a serialized YAML string.""" - yaml = yaml_no_ts() - result = yaml.load(string) + result = yaml_no_ts().load(string) return load_document_by_yaml(result, uri, loadingOptions, id_, load_all) @@ -403,11 +402,7 @@ def save( def is_process(v: Any) -> bool: """Test to see if the object is a CWL v1.x Python Process object.""" - return ( - isinstance(v, cwl_v1_0.Process) - or isinstance(v, cwl_v1_1.Process) - or isinstance(v, cwl_v1_2.Process) - ) + return isinstance(v, cwl_v1_0.Process | cwl_v1_1.Process | cwl_v1_2.Process) def version_split(version: str) -> MutableSequence[int]: diff --git a/cwl_utils/parser/cwl_v1_0_utils.py b/cwl_utils/parser/cwl_v1_0_utils.py index 050e1960..6a647478 100644 --- a/cwl_utils/parser/cwl_v1_0_utils.py +++ b/cwl_utils/parser/cwl_v1_0_utils.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 import hashlib import logging -import os from collections import namedtuple from collections.abc import MutableMapping, MutableSequence from io import StringIO +from pathlib import Path from typing import IO, Any, Union, cast from urllib.parse import urldefrag @@ -83,8 +83,7 @@ def _compare_type(type1: Any, type2: Any) -> bool: if not any(_compare_type(t1, t2) for t2 in type2): return False return True - case _: - return bool(type1 == type2) + return bool(type1 == type2) def _inputfile_load( @@ -102,7 +101,7 @@ def _inputfile_load( doc_url, frg = urldefrag(url) text = loadingOptions.fetcher.fetch_text(doc_url) textIO = StringIO(text) - textIO.name = str(doc_url) + textIO.name = doc_url yaml = yaml_no_ts() result = yaml.load(textIO) add_lc_filename(result, doc_url) @@ -160,7 +159,7 @@ def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: except for 'null'. In strict comparison, all source types must match at least one sink type. """ - if src == "Any" or sink == "Any": + if "Any" in (src, sink): return True if isinstance(src, cwl.ArraySchema) and isinstance(sink, cwl.ArraySchema): return can_assign_src_to_sink(src.items, sink.items, strict) @@ -284,11 +283,9 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: "Not allowed to specify outputBinding when using stdout shortcut." ) if clt.stdout is None: - clt.stdout = str( - hashlib.sha1( # nosec - json_dumps(clt.save(), sort_keys=True).encode("utf-8") - ).hexdigest() - ) + clt.stdout = hashlib.sha1( # nosec + json_dumps(clt.save(), sort_keys=True).encode("utf-8") + ).hexdigest() out.type_ = "File" out.outputBinding = cwl.CommandOutputBinding(glob=clt.stdout) elif out.type_ == "stderr": @@ -297,11 +294,9 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: "Not allowed to specify outputBinding when using stderr shortcut." ) if clt.stderr is None: - clt.stderr = str( - hashlib.sha1( # nosec - json_dumps(clt.save(), sort_keys=True).encode("utf-8") - ).hexdigest() - ) + clt.stderr = hashlib.sha1( # nosec + json_dumps(clt.save(), sort_keys=True).encode("utf-8") + ).hexdigest() out.type_ = "File" out.outputBinding = cwl.CommandOutputBinding(glob=clt.stderr) @@ -313,7 +308,7 @@ def load_inputfile( ) -> Any: """Load a CWL v1.0 input file from a serialized YAML string or a YAML object.""" if baseuri is None: - baseuri = cwl.file_uri(os.getcwd()) + "/" + baseuri = cwl.file_uri(str(Path.cwd())) + "/" if loadingOptions is None: loadingOptions = cwl.LoadingOptions() @@ -331,8 +326,7 @@ def load_inputfile_by_string( loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.0 input file from a serialized YAML string.""" - yaml = yaml_no_ts() - result = yaml.load(string) + result = yaml_no_ts().load(string) add_lc_filename(result, uri) if loadingOptions is None: @@ -472,8 +466,7 @@ def type_for_source( return merge_flatten_type(new_type) elif isinstance(sourcenames, list) and len(sourcenames) > 1: return cwl.ArraySchema(items=new_type, type_="array") - else: - return new_type + return new_type def param_for_source_id( diff --git a/cwl_utils/parser/cwl_v1_1_utils.py b/cwl_utils/parser/cwl_v1_1_utils.py index 96fd88f8..1d9ea49e 100644 --- a/cwl_utils/parser/cwl_v1_1_utils.py +++ b/cwl_utils/parser/cwl_v1_1_utils.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 import hashlib import logging -import os from collections import namedtuple from collections.abc import MutableMapping, MutableSequence from io import StringIO +from pathlib import Path from typing import IO, Any, Union, cast from urllib.parse import urldefrag @@ -83,8 +83,7 @@ def _compare_type(type1: Any, type2: Any) -> bool: if not any(_compare_type(t1, t2) for t2 in type2): return False return True - case _: - return bool(type1 == type2) + return bool(type1 == type2) def _inputfile_load( @@ -102,7 +101,7 @@ def _inputfile_load( doc_url, frg = urldefrag(url) text = loadingOptions.fetcher.fetch_text(doc_url) textIO = StringIO(text) - textIO.name = str(doc_url) + textIO.name = doc_url yaml = yaml_no_ts() result = yaml.load(textIO) add_lc_filename(result, doc_url) @@ -160,7 +159,7 @@ def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: except for 'null'. In strict comparison, all source types must match at least one sink type. """ - if src == "Any" or sink == "Any": + if "Any" in (src, sink): return True if isinstance(src, cwl.ArraySchema) and isinstance(sink, cwl.ArraySchema): return can_assign_src_to_sink(src.items, sink.items, strict) @@ -284,11 +283,9 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: "Not allowed to specify outputBinding when using stdout shortcut." ) if clt.stdout is None: - clt.stdout = str( - hashlib.sha1( # nosec - json_dumps(clt.save(), sort_keys=True).encode("utf-8") - ).hexdigest() - ) + clt.stdout = hashlib.sha1( # nosec + json_dumps(clt.save(), sort_keys=True).encode("utf-8") + ).hexdigest() out.type_ = "File" out.outputBinding = cwl.CommandOutputBinding(glob=clt.stdout) elif out.type_ == "stderr": @@ -297,11 +294,9 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: "Not allowed to specify outputBinding when using stderr shortcut." ) if clt.stderr is None: - clt.stderr = str( - hashlib.sha1( # nosec - json_dumps(clt.save(), sort_keys=True).encode("utf-8") - ).hexdigest() - ) + clt.stderr = hashlib.sha1( # nosec + json_dumps(clt.save(), sort_keys=True).encode("utf-8") + ).hexdigest() out.type_ = "File" out.outputBinding = cwl.CommandOutputBinding(glob=clt.stderr) for inp in clt.inputs: @@ -329,7 +324,7 @@ def load_inputfile( ) -> Any: """Load a CWL v1.1 input file from a serialized YAML string or a YAML object.""" if baseuri is None: - baseuri = cwl.file_uri(os.getcwd()) + "/" + baseuri = cwl.file_uri(str(Path.cwd())) + "/" if loadingOptions is None: loadingOptions = cwl.LoadingOptions() @@ -347,8 +342,7 @@ def load_inputfile_by_string( loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.1 input file from a serialized YAML string.""" - yaml = yaml_no_ts() - result = yaml.load(string) + result = yaml_no_ts().load(string) add_lc_filename(result, uri) if loadingOptions is None: diff --git a/cwl_utils/parser/cwl_v1_2_utils.py b/cwl_utils/parser/cwl_v1_2_utils.py index 36928969..33733b08 100644 --- a/cwl_utils/parser/cwl_v1_2_utils.py +++ b/cwl_utils/parser/cwl_v1_2_utils.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 import hashlib import logging -import os from collections import namedtuple from collections.abc import MutableMapping, MutableSequence from io import StringIO +from pathlib import Path from typing import IO, Any, Union, cast from urllib.parse import urldefrag @@ -83,8 +83,7 @@ def _compare_type(type1: Any, type2: Any) -> bool: if not any(_compare_type(t1, t2) for t2 in type2): return False return True - case _: - return bool(type1 == type2) + return bool(type1 == type2) def _is_all_output_method_loop_step( @@ -121,7 +120,7 @@ def _inputfile_load( doc_url, frg = urldefrag(url) text = loadingOptions.fetcher.fetch_text(doc_url) textIO = StringIO(text) - textIO.name = str(doc_url) + textIO.name = doc_url yaml = yaml_no_ts() result = yaml.load(textIO) add_lc_filename(result, doc_url) @@ -179,7 +178,7 @@ def can_assign_src_to_sink(src: Any, sink: Any, strict: bool = False) -> bool: except for 'null'. In strict comparison, all source types must match at least one sink type. """ - if src == "Any" or sink == "Any": + if "Any" in (src, sink): return True if isinstance(src, cwl.ArraySchema) and isinstance(sink, cwl.ArraySchema): return can_assign_src_to_sink(src.items, sink.items, strict) @@ -229,7 +228,7 @@ def check_all_types( linkMerge = sink.linkMerge or ( "merge_nested" if len(sourceField) > 1 else None ) - if sink.pickValue in ["first_non_null", "the_only_non_null"]: + if sink.pickValue in ("first_non_null", "the_only_non_null"): linkMerge = None srcs_of_sink = [] for parm_id in sourceField: @@ -365,11 +364,9 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: "Not allowed to specify outputBinding when using stdout shortcut." ) if clt.stdout is None: - clt.stdout = str( - hashlib.sha1( # nosec - json_dumps(clt.save(), sort_keys=True).encode("utf-8") - ).hexdigest() - ) + clt.stdout = hashlib.sha1( # nosec + json_dumps(clt.save(), sort_keys=True).encode("utf-8") + ).hexdigest() out.type_ = "File" out.outputBinding = cwl.CommandOutputBinding(glob=clt.stdout) elif out.type_ == "stderr": @@ -378,11 +375,9 @@ def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None: "Not allowed to specify outputBinding when using stderr shortcut." ) if clt.stderr is None: - clt.stderr = str( - hashlib.sha1( # nosec - json_dumps(clt.save(), sort_keys=True).encode("utf-8") - ).hexdigest() - ) + clt.stderr = hashlib.sha1( # nosec + json_dumps(clt.save(), sort_keys=True).encode("utf-8") + ).hexdigest() out.type_ = "File" out.outputBinding = cwl.CommandOutputBinding(glob=clt.stderr) for inp in clt.inputs: @@ -410,7 +405,7 @@ def load_inputfile( ) -> Any: """Load a CWL v1.2 input file from a serialized YAML string or a YAML object.""" if baseuri is None: - baseuri = cwl.file_uri(os.getcwd()) + "/" + baseuri = cwl.file_uri(str(Path.cwd())) + "/" if loadingOptions is None: loadingOptions = cwl.LoadingOptions() @@ -428,8 +423,7 @@ def load_inputfile_by_string( loadingOptions: cwl.LoadingOptions | None = None, ) -> Any: """Load a CWL v1.2 input file from a serialized YAML string.""" - yaml = yaml_no_ts() - result = yaml.load(string) + result = yaml_no_ts().load(string) add_lc_filename(result, uri) if loadingOptions is None: @@ -545,7 +539,7 @@ def type_for_source( new_type = merge_flatten_type(new_type) if pickValue is not None: if isinstance(new_type, cwl.ArraySchema): - if pickValue in ["first_non_null", "the_only_non_null"]: + if pickValue in ("first_non_null", "the_only_non_null"): new_type = new_type.items return new_type new_type = [] @@ -576,7 +570,7 @@ def type_for_source( new_type = cwl.ArraySchema(items=new_type, type_="array") if pickValue is not None: if isinstance(new_type, cwl.ArraySchema): - if pickValue in ["first_non_null", "the_only_non_null"]: + if pickValue in ("first_non_null", "the_only_non_null"): new_type = new_type.items return new_type diff --git a/cwl_utils/parser/utils.py b/cwl_utils/parser/utils.py index 70c5f82d..1823fa9b 100644 --- a/cwl_utils/parser/utils.py +++ b/cwl_utils/parser/utils.py @@ -2,7 +2,6 @@ import copy import logging -import os from collections.abc import MutableSequence from pathlib import Path from types import ModuleType @@ -62,7 +61,7 @@ def load_inputfile_by_uri( if version is None: raise ValidationException("could not get the cwlVersion") - baseuri = str(real_path) + baseuri: str = real_path if loadingOptions is None: match version: @@ -89,7 +88,7 @@ def load_inputfile( ) -> Any: """Load a CWL input file from a serialized YAML string or a YAML object.""" if baseuri is None: - baseuri = cwl_v1_0.file_uri(os.getcwd()) + "/" + baseuri = cwl_v1_0.file_uri(str(Path.cwd())) + "/" if isinstance(doc, str): return load_inputfile_by_string(version, doc, baseuri, loadingOptions) return load_inputfile_by_yaml(version, doc, baseuri, loadingOptions) @@ -102,8 +101,7 @@ def load_inputfile_by_string( loadingOptions: LoadingOptions | None = None, ) -> Any: """Load a CWL input file from a serialized YAML string.""" - yaml = yaml_no_ts() - result = yaml.load(string) + result = yaml_no_ts().load(string) return load_inputfile_by_yaml(version, result, uri, loadingOptions) @@ -147,8 +145,7 @@ def load_step( loadingOptions=step.loadingOptions, ) return cast(Process, step_run) - else: - return cast(Process, copy.deepcopy(step.run)) + return cast(Process, copy.deepcopy(step.run)) def static_checker(workflow: cwl_utils.parser.Workflow) -> None: diff --git a/cwl_utils/sandboxjs.py b/cwl_utils/sandboxjs.py index 83ad3b9a..c0f0f644 100644 --- a/cwl_utils/sandboxjs.py +++ b/cwl_utils/sandboxjs.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Safe execution of CWL Expressions in a NodeJS sandbox.""" + import collections import errno import glob @@ -11,8 +12,10 @@ import threading from abc import ABC, abstractmethod from collections.abc import Awaitable, Mapping, MutableMapping, MutableSequence +from contextlib import suppress from importlib.resources import files from io import BytesIO +from pathlib import Path from typing import Any, Deque, cast from schema_salad.utils import json_dumps @@ -91,7 +94,7 @@ def __init__( self.processes_to_kill: Deque[subprocess.Popen[str]] = collections.deque() def __del__(self) -> None: - try: + with suppress(TypeError): while self.processes_to_kill: process = self.processes_to_kill.popleft() if isinstance(process.args, MutableSequence): @@ -102,7 +105,7 @@ def __del__(self) -> None: str(arg).split("=")[1] for arg in args if "--cidfile" in str(arg) ] if cidfile: # Try to be nice - try: + with suppress(FileNotFoundError): with open(cidfile[0]) as inp_stream: p = subprocess.Popen( # nosec [args[0], "kill", inp_stream.read()], @@ -112,17 +115,11 @@ def __del__(self) -> None: p.wait(timeout=10) except subprocess.TimeoutExpired: p.kill() - except FileNotFoundError: - pass if process.stdin: process.stdin.close() - try: + with suppress(subprocess.TimeoutExpired): process.wait(10) - except subprocess.TimeoutExpired: - pass process.kill() - except TypeError: - pass def check_js_threshold_version(self, working_alias: str) -> bool: """ @@ -204,11 +201,9 @@ def exec_js_process( def terminate() -> None: """Kill the node process if it exceeds timeout limit.""" - try: + with suppress(OSError): killed.append(True) nodejs.kill() - except OSError: - pass timer = threading.Timer(timeout, terminate) timer.daemon = True @@ -299,7 +294,7 @@ def new_js_proc( except (subprocess.CalledProcessError, OSError): pass - if nodejs is None or nodejs is not None and required_node_version is False: + if nodejs is None or nodejs is not None and not required_node_version: try: nodeimg = "node:alpine" if container_engine == "singularity": @@ -323,7 +318,7 @@ def new_js_proc( ) else: singularityimgs = glob.glob( - os.getcwd() + "/node_alpine.sif" + str(Path.cwd() / "node_alpine.sif") ) if singularityimgs: nodeimg = singularityimgs[0] @@ -356,9 +351,9 @@ def new_js_proc( if force_docker_pull: nodejs_pull_commands.append("--force") nodejs_pull_commands.append(nodeimg) - cwd = singularity_cache if singularity_cache else os.getcwd() + cwd = singularity_cache or Path.cwd() nodejsimg = subprocess.check_output( # nosec - nodejs_pull_commands, text=True, cwd=cwd + nodejs_pull_commands, text=True, cwd=str(cwd) ) _logger.debug( "Pulled Docker image %s %s using %s", @@ -439,7 +434,7 @@ def new_js_proc( ) # docker failed, but nodejs is installed on system but the version is below the required version - if docker is False and required_node_version is False: + if not docker and not required_node_version: raise JavascriptException( "NodeJSEngine requires minimum v{} version of Node.js engine.".format( self.minimum_node_version_str diff --git a/cwl_utils/singularity.py b/cwl_utils/singularity.py index 745650ee..03797a1b 100644 --- a/cwl_utils/singularity.py +++ b/cwl_utils/singularity.py @@ -25,8 +25,7 @@ def get_version() -> tuple[list[int], str]: - A tuple with major and minor version numbers as integer. - A string with the name of the singularity flavor. """ - global _SINGULARITY_VERSION # pylint: disable=global-statement - global _SINGULARITY_FLAVOR # pylint: disable=global-statement + global _SINGULARITY_VERSION, _SINGULARITY_FLAVOR # pylint: disable=global-statement if _SINGULARITY_VERSION is None: version_output = check_output( # nosec ["singularity", "--version"], text=True diff --git a/cwl_utils/utils.py b/cwl_utils/utils.py index 86b68dfc..bd0a84c4 100644 --- a/cwl_utils/utils.py +++ b/cwl_utils/utils.py @@ -10,6 +10,7 @@ from collections.abc import MutableMapping, MutableSequence from copy import deepcopy from io import StringIO +from pathlib import Path from typing import Any from urllib.parse import urlparse @@ -41,7 +42,7 @@ def _is_github_symbolic_link(base_url: urllib.parse.ParseResult, contents: str) https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py """ - if base_url.scheme in ["file://", ""]: + if base_url.scheme in ("file://", ""): return False idx = contents.find("\n") @@ -89,7 +90,7 @@ def load_linked_file( """From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py .""" new_url = resolved_path(base_url, link) - if new_url.scheme in ["file://", ""]: + if new_url.scheme in ("file://", ""): contents = pathlib.Path(new_url.path).read_text() else: try: @@ -189,7 +190,7 @@ def resolved_path( elif link_url.scheme == "": # Relative path, can be local or remote - if base_url.scheme in ["file://", ""]: + if base_url.scheme in ("file://", ""): # Local relative path if link == "": return base_url @@ -214,9 +215,9 @@ def singularity_supports_userns() -> bool: global _USERNS # pylint: disable=global-statement if _USERNS is None: try: - hello_image = os.path.join(os.path.dirname(__file__), "hello.simg") + hello_image = Path(os.path.dirname(__file__), "hello.simg") result = subprocess.Popen( # nosec - ["singularity", "exec", "--userns", hello_image, "true"], + ["singularity", "exec", "--userns", str(hello_image), "true"], stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, universal_newlines=True, @@ -252,9 +253,7 @@ def to_pascal_case(name: str) -> str: :param name: :return: """ - return "".join( - map(lambda word: word.capitalize(), name.replace("_", "-").split("-")) - ) + return "".join(map(str.capitalize, name.replace("_", "-").split("-"))) def sanitise_schema_field( @@ -345,7 +344,7 @@ def sanitise_schema_field( | cwl_v1_2.InputRecordSchema() ): return schema_field_item - case {"type": list() as field_item_type}: + case {"type": list(field_item_type)}: if "null" in field_item_type: required = False schema_field_item["type"] = list( @@ -357,12 +356,12 @@ def sanitise_schema_field( # Recursively get items schema_field_item["type"] = list( map( - lambda field_subtypes: sanitise_schema_field(field_subtypes), + sanitise_schema_field, schema_field_item.get("type", []), ) ) - case {"type": str() as field_item_type}: + case {"type": str(field_item_type)}: if field_item_type.endswith("?"): required = False schema_field_item["type"] = schema_field_item.get("type", "").replace( @@ -411,8 +410,7 @@ def is_uri(uri: str) -> bool: """ if not urlparse(uri).scheme == "": return True - else: - return False + return False def is_local_uri(uri: str) -> bool: diff --git a/tests/test_etools_to_clt.py b/tests/test_etools_to_clt.py index 9a2da6e9..749cd372 100644 --- a/tests/test_etools_to_clt.py +++ b/tests/test_etools_to_clt.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 """Test the CWL Expression refactoring tool.""" -import os import shutil import sys import tarfile @@ -262,4 +261,4 @@ def cwl_v1_0_dir( else: tf.extractall(path=tmp_path) yield str(tmp_path / "common-workflow-language-1.0.2") - shutil.rmtree(os.path.join(tmp_path)) + shutil.rmtree(Path(tmp_path)) diff --git a/tests/test_format.py b/tests/test_format.py index 190d1a3a..96af3a5b 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -38,9 +38,9 @@ def _load_format(fetchpath: Path) -> Graph: fetchurl = fetchpath.as_uri() content = fetcher.fetch_text(fetchurl) graph = Graph() - for fmt in ["xml", "turtle", "rdfa"]: + for fmt in ("xml", "turtle", "rdfa"): try: - graph.parse(data=content, format=fmt, publicID=str(fetchurl)) + graph.parse(data=content, format=fmt, publicID=fetchurl) break except (xml.sax.SAXParseException, TypeError, BadSyntax): pass diff --git a/tests/test_inputs_schema_gen.py b/tests/test_inputs_schema_gen.py index 00091b4c..09f0d2c8 100644 --- a/tests/test_inputs_schema_gen.py +++ b/tests/test_inputs_schema_gen.py @@ -58,9 +58,7 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None: f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" ) - yaml = YAML() - - input_obj = yaml.load(inputs_path) + input_obj = YAML().load(inputs_path) try: validate(input_obj, json_schema) @@ -86,9 +84,7 @@ def test_cwl_inputs_to_jsonschema_fails() -> None: f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" ) - yaml = YAML() - - input_obj = yaml.load(inputs_path) + input_obj = YAML().load(inputs_path) # We expect this to fail with pytest.raises(ValidationError): diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index 316d7b72..73d0ca1d 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -71,7 +71,7 @@ def hide_nodejs(temp_dir: Path) -> str: new_dir.mkdir() for entry in os.listdir(dirname): if entry not in ("nodejs", "node"): - os.symlink(os.path.join(dirname, entry), new_dir / entry) + os.symlink(Path(dirname, entry), new_dir / entry) paths.append(str(new_dir)) return ":".join(paths)