From f6d81982924398c07c0ebd3f2b057f2337b43de9 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 26 Aug 2022 17:08:32 -0400 Subject: [PATCH 01/28] Experimental fast validator code path, using cwl-utils Requires schema salad codegen improvements, these are pending On a very large workflow I was testing with, the validation time went 120 seconds to 20 seconds. This is a work in progress. --- cwltool/argparser.py | 7 ++++++ cwltool/context.py | 2 ++ cwltool/load_tool.py | 56 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 8dba4ce28..8057bcbd7 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -579,6 +579,13 @@ def arg_parser() -> argparse.ArgumentParser: default=True, help=argparse.SUPPRESS, ) + parser.add_argument( + "--fast-validator", + dest="fast_validator", + action="store_true", + default=False, + help=argparse.SUPPRESS, + ) reggroup = parser.add_mutually_exclusive_group() reggroup.add_argument( diff --git a/cwltool/context.py b/cwltool/context.py index ef0a7e4f6..25a76ee51 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -103,6 +103,8 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.singularity = False # type: bool self.podman = False # type: bool self.eval_timeout = 60 # type: float + self.codegen_idx = {} + self.fast_validator = False super().__init__(kwargs) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 8f942e5b4..9964c44a6 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -40,6 +40,9 @@ from .update import ALLUPDATES from .utils import CWLObjectType, ResolverType, visit_class +import cwl_utils.parser.cwl_v1_2 +import cwl_utils.parser.cwl_v1_2_utils + jobloaderctx = { "cwl": "https://w3id.org/cwl/cwl#", "cwltool": "http://commonwl.org/cwltool#", @@ -263,6 +266,45 @@ def _add_blank_ids( ) ) +def _fast_validator_convert_stdstreams_to_files(processobj) -> None: + if isinstance(processobj, cwl_utils.parser.cwl_v1_2.CommandLineTool): + cwl_utils.parser.cwl_v1_2_utils.convert_stdstreams_to_files(processobj) + elif isinstance(processobj, cwl_utils.parser.cwl_v1_2.Workflow): + for st in processobj.steps: + _fast_validator_convert_stdstreams_to_files(st.run) + elif isinstance(processobj, MutableSequence): + for p in processobj: + _fast_validator_convert_stdstreams_to_files(p) + + +def fast_validator(workflowobj, fileuri, uri, loadingContext: LoadingContext): + lopt = cwl_utils.parser.cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri) + + if uri not in loadingContext.codegen_idx: + cwl_utils.parser.cwl_v1_2.load_document_with_metadata(workflowobj, fileuri, loadingOptions=lopt, addl_metadata_fields=("id", "cwlVersion")) + + objects, loadopt = loadingContext.codegen_idx[uri] + + _fast_validator_convert_stdstreams_to_files(objects) + + processobj = cwl_utils.parser.cwl_v1_2.save(objects, relative_uris=False) + + metadata = {} + metadata["id"] = loadopt.fileuri + + if loadopt.namespaces: + metadata["$namespaces"] = loadopt.namespaces + if loadopt.schemas: + metadata["$schemas"] = loadopt.schemas + if loadopt.baseuri: + metadata["$base"] = loadopt.baseuri + for k,v in loadopt.addl_metadata.items(): + if isinstance(processobj, MutableMapping) and k in processobj: + metadata[k] = processobj[k] + else: + metadata[k] = v + + return cmap(processobj), cmap(metadata) def resolve_and_validate_document( loadingContext: LoadingContext, @@ -381,8 +423,15 @@ def resolve_and_validate_document( if cwlVersion == "v1.0": _add_blank_ids(workflowobj) - document_loader.resolve_all(workflowobj, fileuri) - processobj, metadata = document_loader.resolve_ref(uri) + if cwlVersion != "v1.2": + loadingContext.fast_validator = False + + if loadingContext.fast_validator: + processobj, metadata = fast_validator(workflowobj, fileuri, uri, loadingContext) + else: + document_loader.resolve_all(workflowobj, fileuri) + processobj, metadata = document_loader.resolve_ref(uri) + if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.") @@ -405,7 +454,8 @@ def resolve_and_validate_document( if isinstance(processobj, CommentedMap): uri = processobj["id"] - _convert_stdstreams_to_files(workflowobj) + if not loadingContext.fast_validator: + _convert_stdstreams_to_files(workflowobj) if isinstance(jobobj, CommentedMap): loadingContext.jobdefaults = jobobj From b145e3c5db6865e96ad2aeb7b5de8b472a1de58d Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sat, 27 Aug 2022 14:09:44 -0400 Subject: [PATCH 02/28] Load the schema graph --- cwltool/load_tool.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 9964c44a6..778c0de07 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -304,6 +304,8 @@ def fast_validator(workflowobj, fileuri, uri, loadingContext: LoadingContext): else: metadata[k] = v + loadingContext.loader.graph += loadopt.graph + return cmap(processobj), cmap(metadata) def resolve_and_validate_document( @@ -420,6 +422,8 @@ def resolve_and_validate_document( doc_cache=loadingContext.doc_cache, ) + loadingContext.loader = document_loader + if cwlVersion == "v1.0": _add_blank_ids(workflowobj) @@ -460,7 +464,6 @@ def resolve_and_validate_document( if isinstance(jobobj, CommentedMap): loadingContext.jobdefaults = jobobj - loadingContext.loader = document_loader loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata @@ -496,7 +499,11 @@ def make_tool( """Make a Python CWL object.""" if loadingContext.loader is None: raise ValueError("loadingContext must have a loader") - resolveduri, metadata = loadingContext.loader.resolve_ref(uri) + + if loadingContext.fast_validator: + resolveduri, metadata = fast_validator(None, None, uri, loadingContext) + else: + resolveduri, metadata = loadingContext.loader.resolve_ref(uri) processobj = None if isinstance(resolveduri, MutableSequence): From 3cb3d42db1e288a14d61bb1e089e58ff4df6d160 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 31 Aug 2022 19:47:26 +0200 Subject: [PATCH 03/28] DO NOT MERGE: use cwl-utils from draft PR --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 61213db5a..87c007da8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ pydot>=1.4.1 argcomplete>=1.12.0 pyparsing != 3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 pyparsing < 3;python_version<='3.6' # breaks --print-dot -cwl-utils>=0.15 +https://github.com/common-workflow-language/cwl-utils/archive/codegen-fixes.zip diff --git a/setup.py b/setup.py index ad23a1166..a9d02da26 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "pyparsing < 3 ;python_version<='3.6'", # breaks --print-dot (pydot) "argcomplete", - "cwl-utils >= 0.15", + "cwl-utils @ https://github.com/common-workflow-language/cwl-utils/archive/codegen-fixes.zip", ], extras_require={ "deps": ["galaxy-tool-util >= 21.1.0"], From a5a1b85892ce3e6818180056f17f7aa327a21f7e Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 31 Aug 2022 20:07:01 +0200 Subject: [PATCH 04/28] improve type hints --- cwltool/context.py | 19 +++++++++++++++--- cwltool/load_tool.py | 47 +++++++++++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/cwltool/context.py b/cwltool/context.py index 25a76ee51..42d8dd28f 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -4,7 +4,18 @@ import shutil import tempfile import threading -from typing import IO, Any, Callable, Dict, Iterable, List, Optional, TextIO, Union +from typing import ( + IO, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + TextIO, + Tuple, + Union, +) # move to a regular typing import when Python 3.3-3.6 is no longer supported from ruamel.yaml.comments import CommentedMap @@ -23,6 +34,8 @@ from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType if TYPE_CHECKING: + from cwl_utils.parser.cwl_v1_2 import LoadingOptions + from .process import Process from .provenance import ResearchObject # pylint: disable=unused-import from .provenance_profile import ProvenanceProfile @@ -102,8 +115,8 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.relax_path_checks = False # type: bool self.singularity = False # type: bool self.podman = False # type: bool - self.eval_timeout = 60 # type: float - self.codegen_idx = {} + self.eval_timeout: float = 60 + self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} self.fast_validator = False super().__init__(kwargs) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 778c0de07..7135ee2e6 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -19,6 +19,7 @@ cast, ) +from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader, file_uri @@ -40,9 +41,6 @@ from .update import ALLUPDATES from .utils import CWLObjectType, ResolverType, visit_class -import cwl_utils.parser.cwl_v1_2 -import cwl_utils.parser.cwl_v1_2_utils - jobloaderctx = { "cwl": "https://w3id.org/cwl/cwl#", "cwltool": "http://commonwl.org/cwltool#", @@ -266,10 +264,13 @@ def _add_blank_ids( ) ) -def _fast_validator_convert_stdstreams_to_files(processobj) -> None: - if isinstance(processobj, cwl_utils.parser.cwl_v1_2.CommandLineTool): - cwl_utils.parser.cwl_v1_2_utils.convert_stdstreams_to_files(processobj) - elif isinstance(processobj, cwl_utils.parser.cwl_v1_2.Workflow): + +def _fast_validator_convert_stdstreams_to_files( + processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]] +) -> None: + if isinstance(processobj, cwl_v1_2.CommandLineTool): + cwl_v1_2_utils.convert_stdstreams_to_files(processobj) + elif isinstance(processobj, cwl_v1_2.Workflow): for st in processobj.steps: _fast_validator_convert_stdstreams_to_files(st.run) elif isinstance(processobj, MutableSequence): @@ -277,19 +278,29 @@ def _fast_validator_convert_stdstreams_to_files(processobj) -> None: _fast_validator_convert_stdstreams_to_files(p) -def fast_validator(workflowobj, fileuri, uri, loadingContext: LoadingContext): - lopt = cwl_utils.parser.cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri) +def fast_validator( + workflowobj: Union[CommentedMap, CommentedSeq, None], + fileuri: Optional[str], + uri: str, + loadingContext: LoadingContext, +) -> Tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: + lopt = cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri) if uri not in loadingContext.codegen_idx: - cwl_utils.parser.cwl_v1_2.load_document_with_metadata(workflowobj, fileuri, loadingOptions=lopt, addl_metadata_fields=("id", "cwlVersion")) + cwl_v1_2.load_document_with_metadata( + workflowobj, + fileuri, + loadingOptions=lopt, + addl_metadata_fields=["id", "cwlVersion"], + ) objects, loadopt = loadingContext.codegen_idx[uri] _fast_validator_convert_stdstreams_to_files(objects) - processobj = cwl_utils.parser.cwl_v1_2.save(objects, relative_uris=False) + processobj = cwl_v1_2.save(objects, relative_uris=False) - metadata = {} + metadata: Dict[str, Any] = {} metadata["id"] = loadopt.fileuri if loadopt.namespaces: @@ -298,15 +309,19 @@ def fast_validator(workflowobj, fileuri, uri, loadingContext: LoadingContext): metadata["$schemas"] = loadopt.schemas if loadopt.baseuri: metadata["$base"] = loadopt.baseuri - for k,v in loadopt.addl_metadata.items(): + for k, v in loadopt.addl_metadata.items(): if isinstance(processobj, MutableMapping) and k in processobj: metadata[k] = processobj[k] else: metadata[k] = v - loadingContext.loader.graph += loadopt.graph + if loadingContext.loader: + loadingContext.loader.graph += loadopt.graph + + return cast(Union[CommentedMap, CommentedSeq], cmap(processobj)), cast( + CommentedMap, cmap(metadata) + ) - return cmap(processobj), cmap(metadata) def resolve_and_validate_document( loadingContext: LoadingContext, @@ -500,7 +515,7 @@ def make_tool( if loadingContext.loader is None: raise ValueError("loadingContext must have a loader") - if loadingContext.fast_validator: + if loadingContext.fast_validator and isinstance(uri, str): resolveduri, metadata = fast_validator(None, None, uri, loadingContext) else: resolveduri, metadata = loadingContext.loader.resolve_ref(uri) From b00b17ef8dcd73f00e4be49d1bc6b326109ddcf5 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 1 Sep 2022 12:10:37 -0400 Subject: [PATCH 05/28] Fix load_tool typing --- cwltool/load_tool.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 7135ee2e6..f146162d1 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -298,6 +298,8 @@ def fast_validator( _fast_validator_convert_stdstreams_to_files(objects) + processobj: Union[MutableMapping[str, Any], MutableSequence[Any], float, str, None] + processobj = cwl_v1_2.save(objects, relative_uris=False) metadata: Dict[str, Any] = {} @@ -318,9 +320,10 @@ def fast_validator( if loadingContext.loader: loadingContext.loader.graph += loadopt.graph - return cast(Union[CommentedMap, CommentedSeq], cmap(processobj)), cast( - CommentedMap, cmap(metadata) - ) + return cast( + Union[CommentedMap, CommentedSeq], + cmap(cast(Union[Dict[str, Any], List[Any]], processobj)), + ), cast(CommentedMap, cmap(metadata)) def resolve_and_validate_document( @@ -354,6 +357,8 @@ def resolve_and_validate_document( fileuri = urllib.parse.urldefrag(uri)[0] + metadata: CWLObjectType + cwlVersion = loadingContext.metadata.get("cwlVersion") if not cwlVersion: cwlVersion = workflowobj.get("cwlVersion") @@ -426,7 +431,7 @@ def resolve_and_validate_document( if isinstance(avsc_names, Exception): raise avsc_names - processobj = None # type: Optional[ResolveType] + processobj: Union[CommentedMap, CommentedSeq, int, float, str, None] document_loader = Loader( sch_document_loader.ctx, schemagraph=sch_document_loader.graph, @@ -515,6 +520,9 @@ def make_tool( if loadingContext.loader is None: raise ValueError("loadingContext must have a loader") + resolveduri: Union[float, str, CommentedMap, CommentedSeq, None] + metadata: CWLObjectType + if loadingContext.fast_validator and isinstance(uri, str): resolveduri, metadata = fast_validator(None, None, uri, loadingContext) else: From 3adbcbc55b3736769665092097a857750e828c8c Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 2 Sep 2022 17:01:23 +0200 Subject: [PATCH 06/28] GA: run the conformance tests with the fast validator --- .github/workflows/ci-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5b89bbc61..1a76b3769 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -121,6 +121,11 @@ jobs: matrix: cwl-version: [v1.0, v1.1, v1.2] container: [docker, singularity, podman] + extras: [""] + include: + - cwl-version: v1.2 + container: docker + extras: "--fast-validator" steps: - uses: actions/checkout@v3 @@ -141,6 +146,7 @@ jobs: version: ${{ matrix.cwl-version }} container: ${{ matrix.container }} spec_branch: main + CWLTOOL_OPTIONS: ${{ matrix.extras }} run: ./conformance-test.sh release_test: From 772a11c6f20f19b83368c37d0cabacb5324ec992 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 2 Sep 2022 17:17:54 +0200 Subject: [PATCH 07/28] conformance testing: use CWLTOOL_OPTIONS --- conformance-test.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/conformance-test.sh b/conformance-test.sh index 90075a7fb..016edfe22 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -92,15 +92,15 @@ chmod a+x "${CWLTOOL_WITH_COV}" unset exclusions declare -a exclusions -EXTRA="--parallel" +CWLTOOL_OPTIONS+=" --parallel" # shellcheck disable=SC2154 if [[ "$version" = *dev* ]] then - EXTRA+=" --enable-dev" + CWLTOOL_OPTIONS+=" --enable-dev" fi if [[ "$container" = "singularity" ]]; then - EXTRA+=" --singularity" + CWLTOOL_OPTIONS+=" --singularity" # This test fails because Singularity and Docker have # different views on how to deal with this. exclusions+=(docker_entrypoint) @@ -113,13 +113,9 @@ if [[ "$container" = "singularity" ]]; then exclusions+=(stdin_shorcut) fi elif [[ "$container" = "podman" ]]; then - EXTRA+=" --podman" + CWLTOOL_OPTIONS+=" --podman" fi -if [ -n "$EXTRA" ] -then - EXTRA="EXTRA=${EXTRA}" -fi if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] && [[ "$container" = "docker" ]] then rm -Rf conformance @@ -148,10 +144,11 @@ if (( "${#exclusions[*]}" > 0 )); then else EXCLUDE="" fi +export CWLTOOL_OPTIONS # shellcheck disable=SC2086 LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml ${EXCLUDE} \ RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \ - ${DRAFT} "${EXTRA}" \ + ${DRAFT} \ "--classname=py3_${container}" # LC_ALL=C is to work around junit-xml ASCII only bug From d140151c43385fa392fd27e7e6112d036680e4e6 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 2 Sep 2022 17:26:12 +0200 Subject: [PATCH 08/28] CWLTOOL_OPTIONS: ignore an empty string --- cwltool/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cwltool/main.py b/cwltool/main.py index 63e3d8679..e82a8ee6c 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -1011,7 +1011,8 @@ def main( argsl = sys.argv[1:] addl = [] # type: List[str] if "CWLTOOL_OPTIONS" in os.environ: - addl = os.environ["CWLTOOL_OPTIONS"].split(" ") + c_opts = os.environ["CWLTOOL_OPTIONS"].split(" ") + addl = [x for x in c_opts if x != ""] parser = arg_parser() argcomplete.autocomplete(parser) args = parser.parse_args(addl + argsl) From 80271c4da4e6228fb8c883e68df8607b20159a01 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 2 Sep 2022 17:31:22 +0200 Subject: [PATCH 09/28] DO NOT MERGE: container build needs our cwl-util branch --- mypy-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 9118c91a0..22f149328 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,7 +1,7 @@ mypy==0.971 ruamel.yaml>=0.16.0,<0.17.22 schema-salad>=8.2.20211104054942,<9 -cwl-utils>=0.15 +cwl-utils @ https://github.com/common-workflow-language/cwl-utils/archive/codegen-fixes.zip types-requests types-setuptools types-psutil From 3d12d6ecc5db0bc6219bfe296d6bc4d12f5b66a0 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 8 Sep 2022 23:07:20 -0400 Subject: [PATCH 10/28] Rename --fast-validator to --fast-parser Expand 'class' field of hints --- cwltool/argparser.py | 4 ++-- cwltool/context.py | 2 +- cwltool/load_tool.py | 49 ++++++++++++++++++++++++++++++++++---------- requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 4ac4a8f12..2d5246ee0 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -577,8 +577,8 @@ def arg_parser() -> argparse.ArgumentParser: help=argparse.SUPPRESS, ) parser.add_argument( - "--fast-validator", - dest="fast_validator", + "--fast-parser", + dest="fast_parser", action="store_true", default=False, help=argparse.SUPPRESS, diff --git a/cwltool/context.py b/cwltool/context.py index 42d8dd28f..b9fa6d3d6 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -117,7 +117,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.podman = False # type: bool self.eval_timeout: float = 60 self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} - self.fast_validator = False + self.fast_parser = False super().__init__(kwargs) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index f146162d1..d16c62c18 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -265,20 +265,46 @@ def _add_blank_ids( ) -def _fast_validator_convert_stdstreams_to_files( +def _fast_parser_convert_stdstreams_to_files( processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]] ) -> None: if isinstance(processobj, cwl_v1_2.CommandLineTool): cwl_v1_2_utils.convert_stdstreams_to_files(processobj) elif isinstance(processobj, cwl_v1_2.Workflow): for st in processobj.steps: - _fast_validator_convert_stdstreams_to_files(st.run) + _fast_parser_convert_stdstreams_to_files(st.run) elif isinstance(processobj, MutableSequence): for p in processobj: - _fast_validator_convert_stdstreams_to_files(p) + _fast_parser_convert_stdstreams_to_files(p) +def _fast_parser_expand_hint_class( + hints, + loadingOptions: cwl_v1_2.LoadingOptions): -def fast_validator( + for h in hints: + if isinstance(h, MutableMapping) and "class" in h: + for k, v in loadingOptions.namespaces.items(): + if h["class"].startswith(k+":"): + h["class"] = v + h["class"][len(k)+1:] + + +def _fast_parser_handle_hints( + processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], + loadingOptions: cwl_v1_2.LoadingOptions +) -> None: + if isinstance(processobj, (cwl_v1_2.CommandLineTool, cwl_v1_2.Workflow)): + _fast_parser_expand_hint_class(processobj.hints, loadingOptions) + + if isinstance(processobj, cwl_v1_2.Workflow): + for st in processobj.steps: + _fast_parser_expand_hint_class(st.hints, loadingOptions) + _fast_parser_handle_hints(st.run) + elif isinstance(processobj, MutableSequence): + for p in processobj: + _fast_parser_handle_hints(p) + + +def fast_parser( workflowobj: Union[CommentedMap, CommentedSeq, None], fileuri: Optional[str], uri: str, @@ -296,7 +322,8 @@ def fast_validator( objects, loadopt = loadingContext.codegen_idx[uri] - _fast_validator_convert_stdstreams_to_files(objects) + _fast_parser_convert_stdstreams_to_files(objects) + _fast_parser_handle_hints(objects, loadopt) processobj: Union[MutableMapping[str, Any], MutableSequence[Any], float, str, None] @@ -448,10 +475,10 @@ def resolve_and_validate_document( _add_blank_ids(workflowobj) if cwlVersion != "v1.2": - loadingContext.fast_validator = False + loadingContext.fast_parser = False - if loadingContext.fast_validator: - processobj, metadata = fast_validator(workflowobj, fileuri, uri, loadingContext) + if loadingContext.fast_parser: + processobj, metadata = fast_parser(workflowobj, fileuri, uri, loadingContext) else: document_loader.resolve_all(workflowobj, fileuri) processobj, metadata = document_loader.resolve_ref(uri) @@ -478,7 +505,7 @@ def resolve_and_validate_document( if isinstance(processobj, CommentedMap): uri = processobj["id"] - if not loadingContext.fast_validator: + if not loadingContext.fast_parser: _convert_stdstreams_to_files(workflowobj) if isinstance(jobobj, CommentedMap): @@ -523,8 +550,8 @@ def make_tool( resolveduri: Union[float, str, CommentedMap, CommentedSeq, None] metadata: CWLObjectType - if loadingContext.fast_validator and isinstance(uri, str): - resolveduri, metadata = fast_validator(None, None, uri, loadingContext) + if loadingContext.fast_parser and isinstance(uri, str): + resolveduri, metadata = fast_parser(None, None, uri, loadingContext) else: resolveduri, metadata = loadingContext.loader.resolve_ref(uri) diff --git a/requirements.txt b/requirements.txt index 87c007da8..186c3b522 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ pydot>=1.4.1 argcomplete>=1.12.0 pyparsing != 3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 pyparsing < 3;python_version<='3.6' # breaks --print-dot -https://github.com/common-workflow-language/cwl-utils/archive/codegen-fixes.zip +cwl-utils>=0.18 diff --git a/setup.py b/setup.py index 32056a344..8650bdb6f 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "pyparsing < 3 ;python_version<='3.6'", # breaks --print-dot (pydot) "argcomplete", - "cwl-utils @ https://github.com/common-workflow-language/cwl-utils/archive/codegen-fixes.zip", + "cwl-utils >= 0.18", ], extras_require={ "deps": ["galaxy-tool-util >= 22.1.2, <23"], From 3617dd5b8366b949ebff89201007dc9fefac355f Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 8 Sep 2022 23:12:26 -0400 Subject: [PATCH 11/28] mypy & format --- cwltool/load_tool.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index d16c62c18..2b9943e51 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -277,20 +277,22 @@ def _fast_parser_convert_stdstreams_to_files( for p in processobj: _fast_parser_convert_stdstreams_to_files(p) + def _fast_parser_expand_hint_class( - hints, - loadingOptions: cwl_v1_2.LoadingOptions): + hints: Optional[Any], loadingOptions: cwl_v1_2.LoadingOptions +) -> None: - for h in hints: - if isinstance(h, MutableMapping) and "class" in h: - for k, v in loadingOptions.namespaces.items(): - if h["class"].startswith(k+":"): - h["class"] = v + h["class"][len(k)+1:] + if isinstance(hints, MutableSequence): + for h in hints: + if isinstance(h, MutableMapping) and "class" in h: + for k, v in loadingOptions.namespaces.items(): + if h["class"].startswith(k + ":"): + h["class"] = v + h["class"][len(k) + 1 :] def _fast_parser_handle_hints( - processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], - loadingOptions: cwl_v1_2.LoadingOptions + processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], + loadingOptions: cwl_v1_2.LoadingOptions, ) -> None: if isinstance(processobj, (cwl_v1_2.CommandLineTool, cwl_v1_2.Workflow)): _fast_parser_expand_hint_class(processobj.hints, loadingOptions) @@ -298,10 +300,10 @@ def _fast_parser_handle_hints( if isinstance(processobj, cwl_v1_2.Workflow): for st in processobj.steps: _fast_parser_expand_hint_class(st.hints, loadingOptions) - _fast_parser_handle_hints(st.run) + _fast_parser_handle_hints(st.run, loadingOptions) elif isinstance(processobj, MutableSequence): for p in processobj: - _fast_parser_handle_hints(p) + _fast_parser_handle_hints(p, loadingOptions) def fast_parser( From d6226ec025b3c303cff44b34f26141f7d819c1bf Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 9 Sep 2022 09:46:38 -0400 Subject: [PATCH 12/28] Fix mypy-requirements --- mypy-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 22f149328..684853cbe 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,7 +1,7 @@ mypy==0.971 ruamel.yaml>=0.16.0,<0.17.22 schema-salad>=8.2.20211104054942,<9 -cwl-utils @ https://github.com/common-workflow-language/cwl-utils/archive/codegen-fixes.zip +cwl-utils >=0.18 types-requests types-setuptools types-psutil From b503d545465f6532a7b6d3f63c5317a44e6cc76e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 9 Sep 2022 09:53:01 -0400 Subject: [PATCH 13/28] Fix github action --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1a76b3769..30c7d3027 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -125,7 +125,7 @@ jobs: include: - cwl-version: v1.2 container: docker - extras: "--fast-validator" + extras: "--fast-parser" steps: - uses: actions/checkout@v3 From b686e41d552f49639e16224369f85596be8e0093 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 9 Sep 2022 19:18:23 +0200 Subject: [PATCH 14/28] use ResolveType --- cwltool/load_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 2b9943e51..2c2e4b02a 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -460,7 +460,7 @@ def resolve_and_validate_document( if isinstance(avsc_names, Exception): raise avsc_names - processobj: Union[CommentedMap, CommentedSeq, int, float, str, None] + processobj: ResolveType document_loader = Loader( sch_document_loader.ctx, schemagraph=sch_document_loader.graph, From 833b0d6a1d6e3c3cc9d8c1b5ca5fa2ade8e80ffe Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 9 Sep 2022 14:07:46 -0400 Subject: [PATCH 15/28] Add section to README about --fast-parser --- README.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.rst b/README.rst index 4a76a5656..aebae5114 100644 --- a/README.rst +++ b/README.rst @@ -667,6 +667,19 @@ given in the following table; all are optional. +----------------+------------------+----------+------------------------------+ +Enabling Fast Parser (experimental) +----------------------------------- + +For very large workflows, it can spend a lot of time in +initialization, before the first step runs. There is an experimental +flag ``--fast-parser`` which can dramatically reduce the +initialization overhead, however as of this writing it has several limitations: + +- Error reporting in general is worse than the standard parser, you will want to use it with workflows that you know are already correct. + +- It does not check for dangling links (these will become runtime errors instead of loading errors) + +- Several other cases fail, as documented in https://github.com/common-workflow-language/cwltool/pull/1720 =========== Development From d64be75af4f73036a570542dd53e0d0a6c10f801 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 9 Sep 2022 15:22:37 -0400 Subject: [PATCH 16/28] Fast parser expects comment data --- cwltool/update.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cwltool/update.py b/cwltool/update.py index e60c696f9..0e2042fa5 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -124,14 +124,19 @@ def fix_inputBinding(t: CWLObjectType) -> None: upd = upd["$graph"] for proc in aslist(upd): proc.setdefault("hints", CommentedSeq()) + na = CommentedMap([("class", "NetworkAccess"), ("networkAccess", True)]) + na.lc.filename = proc.lc.filename proc["hints"].insert( - 0, CommentedMap([("class", "NetworkAccess"), ("networkAccess", True)]) + 0, na ) + + ll = CommentedMap( + [("class", "LoadListingRequirement"), ("loadListing", "deep_listing")] + ) + ll.lc.filename = proc.lc.filename proc["hints"].insert( 0, - CommentedMap( - [("class", "LoadListingRequirement"), ("loadListing", "deep_listing")] - ), + ll, ) if "cwlVersion" in proc: del proc["cwlVersion"] From 1113260c460efc18a3fa676908a67c640850ed76 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 9 Sep 2022 15:39:50 -0400 Subject: [PATCH 17/28] format --- cwltool/update.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cwltool/update.py b/cwltool/update.py index 0e2042fa5..5723e7db7 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -126,13 +126,11 @@ def fix_inputBinding(t: CWLObjectType) -> None: proc.setdefault("hints", CommentedSeq()) na = CommentedMap([("class", "NetworkAccess"), ("networkAccess", True)]) na.lc.filename = proc.lc.filename - proc["hints"].insert( - 0, na - ) + proc["hints"].insert(0, na) ll = CommentedMap( - [("class", "LoadListingRequirement"), ("loadListing", "deep_listing")] - ) + [("class", "LoadListingRequirement"), ("loadListing", "deep_listing")] + ) ll.lc.filename = proc.lc.filename proc["hints"].insert( 0, From efa5c44351857dcd57732d96d8530d94e255657b Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sat, 10 Sep 2022 12:27:59 -0400 Subject: [PATCH 18/28] Update document_loader index to match fast parser --- cwltool/load_tool.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 2c2e4b02a..3e9fd1cc9 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -516,7 +516,16 @@ def resolve_and_validate_document( loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata + def update_index(pr: CommentedMap) -> None: + if "id" in pr: + document_loader.idx[pr["id"]] = pr + if preprocess_only: + if loadingContext.fast_parser: + # Need to match the document loader's index with the fast parser index + visit_class( + processobj, ("CommandLineTool", "Workflow", "ExpressionTool"), update_index + ) return loadingContext, uri if loadingContext.do_validate: @@ -531,10 +540,6 @@ def resolve_and_validate_document( ) document_loader.idx[processobj["id"]] = processobj - def update_index(pr: CommentedMap) -> None: - if "id" in pr: - document_loader.idx[pr["id"]] = pr - visit_class( processobj, ("CommandLineTool", "Workflow", "ExpressionTool"), update_index ) From acaf81c6b4403d145cd5d9435ab20469dea704eb Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sat, 10 Sep 2022 12:30:51 -0400 Subject: [PATCH 19/28] Fix format --- cwltool/load_tool.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 3e9fd1cc9..1074847ef 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -524,7 +524,9 @@ def update_index(pr: CommentedMap) -> None: if loadingContext.fast_parser: # Need to match the document loader's index with the fast parser index visit_class( - processobj, ("CommandLineTool", "Workflow", "ExpressionTool"), update_index + processobj, + ("CommandLineTool", "Workflow", "ExpressionTool"), + update_index, ) return loadingContext, uri From 0584fe8806ba9d20035e87301fd5a235e982f7f4 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sun, 11 Sep 2022 14:59:55 -0400 Subject: [PATCH 20/28] Make sure everything in $graph gets into the index --- cwltool/load_tool.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 1074847ef..79b46aeb1 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -18,6 +18,7 @@ Union, cast, ) +from functools import partial from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -306,6 +307,11 @@ def _fast_parser_handle_hints( _fast_parser_handle_hints(p, loadingOptions) +def update_index(document_loader: Loader, pr: CommentedMap) -> None: + if "id" in pr: + document_loader.idx[pr["id"]] = pr + + def fast_parser( workflowobj: Union[CommentedMap, CommentedSeq, None], fileuri: Optional[str], @@ -349,6 +355,22 @@ def fast_parser( if loadingContext.loader: loadingContext.loader.graph += loadopt.graph + # Need to match the document loader's index with the fast parser index + # Get the base URI (no fragments) for documents that use $graph + nofrag = urllib.parse.urldefrag(uri)[0] + objects, loadopt = loadingContext.codegen_idx[nofrag] + fileobj = cmap( + cast( + Union[int, float, str, Dict[str, Any], List[Any], None], + cwl_v1_2.save(objects, relative_uris=False), + ) + ) + visit_class( + fileobj, + ("CommandLineTool", "Workflow", "ExpressionTool"), + partial(update_index, loadingContext.loader), + ) + return cast( Union[CommentedMap, CommentedSeq], cmap(cast(Union[Dict[str, Any], List[Any]], processobj)), @@ -516,18 +538,7 @@ def resolve_and_validate_document( loadingContext.avsc_names = avsc_names loadingContext.metadata = metadata - def update_index(pr: CommentedMap) -> None: - if "id" in pr: - document_loader.idx[pr["id"]] = pr - if preprocess_only: - if loadingContext.fast_parser: - # Need to match the document loader's index with the fast parser index - visit_class( - processobj, - ("CommandLineTool", "Workflow", "ExpressionTool"), - update_index, - ) return loadingContext, uri if loadingContext.do_validate: @@ -543,7 +554,9 @@ def update_index(pr: CommentedMap) -> None: document_loader.idx[processobj["id"]] = processobj visit_class( - processobj, ("CommandLineTool", "Workflow", "ExpressionTool"), update_index + processobj, + ("CommandLineTool", "Workflow", "ExpressionTool"), + partial(update_index, document_loader), ) return loadingContext, uri From 46f53bcdd4e0a46adb26ba319d6c6f421495f15a Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sun, 11 Sep 2022 22:16:35 -0400 Subject: [PATCH 21/28] Don't crash update if filename field is missing --- cwltool/update.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cwltool/update.py b/cwltool/update.py index 5723e7db7..04a2d95d5 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -125,13 +125,19 @@ def fix_inputBinding(t: CWLObjectType) -> None: for proc in aslist(upd): proc.setdefault("hints", CommentedSeq()) na = CommentedMap([("class", "NetworkAccess"), ("networkAccess", True)]) - na.lc.filename = proc.lc.filename + + if hasattr(proc.lc, "filename"): + comment_filename = proc.lc.filename + else: + comment_filename = "" + na.lc.filename = comment_filename + proc["hints"].insert(0, na) ll = CommentedMap( [("class", "LoadListingRequirement"), ("loadListing", "deep_listing")] ) - ll.lc.filename = proc.lc.filename + ll.lc.filename = comment_filename proc["hints"].insert( 0, ll, From 03b2f0207d2d08e15028d607e81709e90942cb59 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sun, 11 Sep 2022 23:05:07 -0400 Subject: [PATCH 22/28] Add loadingContext.skip_resolve_all with note --- cwltool/context.py | 1 + cwltool/load_tool.py | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cwltool/context.py b/cwltool/context.py index b9fa6d3d6..4afb1ea10 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -118,6 +118,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.eval_timeout: float = 60 self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} self.fast_parser = False + self.skip_resolve_all = False super().__init__(kwargs) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 79b46aeb1..4262d9918 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -501,7 +501,23 @@ def resolve_and_validate_document( if cwlVersion != "v1.2": loadingContext.fast_parser = False - if loadingContext.fast_parser: + if loadingContext.skip_resolve_all: + # Some integrations (e.g. Arvados) loads documents, makes + # in-memory changes to them (which are applied to the objects + # in the document_loader index), and then sends them back + # through the loading machinery. + # + # In this case, the functions of resolve_all() have already + # happened. Because resolve_all() is expensive, we don't want + # to do it again if it's going to be a no-op, so the + # skip_resolve_all flag tells us just to use the document + # as-is from the loader index. + # + # Note that at the moment, fast_parser code path is considered + # functionally the same as resolve_all() for this case. + # + processobj, metadata = document_loader.resolve_ref(uri) + elif loadingContext.fast_parser: processobj, metadata = fast_parser(workflowobj, fileuri, uri, loadingContext) else: document_loader.resolve_all(workflowobj, fileuri) @@ -572,7 +588,7 @@ def make_tool( resolveduri: Union[float, str, CommentedMap, CommentedSeq, None] metadata: CWLObjectType - if loadingContext.fast_parser and isinstance(uri, str): + if loadingContext.fast_parser and isinstance(uri, str) and not loadingContext.skip_resolve_all: resolveduri, metadata = fast_parser(None, None, uri, loadingContext) else: resolveduri, metadata = loadingContext.loader.resolve_ref(uri) From 7630a512154d98c58a31385d80860e4836900f6d Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sun, 11 Sep 2022 23:13:17 -0400 Subject: [PATCH 23/28] Fix format --- cwltool/load_tool.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 4262d9918..22646b9cc 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -588,7 +588,11 @@ def make_tool( resolveduri: Union[float, str, CommentedMap, CommentedSeq, None] metadata: CWLObjectType - if loadingContext.fast_parser and isinstance(uri, str) and not loadingContext.skip_resolve_all: + if ( + loadingContext.fast_parser + and isinstance(uri, str) + and not loadingContext.skip_resolve_all + ): resolveduri, metadata = fast_parser(None, None, uri, loadingContext) else: resolveduri, metadata = loadingContext.loader.resolve_ref(uri) From db08a47c866a6db173a29f62a1d5b678350e4bac Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 13 Sep 2022 12:13:06 -0400 Subject: [PATCH 24/28] Update README.rst Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index aebae5114..87313bff6 100644 --- a/README.rst +++ b/README.rst @@ -670,7 +670,7 @@ given in the following table; all are optional. Enabling Fast Parser (experimental) ----------------------------------- -For very large workflows, it can spend a lot of time in +For very large workflows, `cwltool` can spend a lot of time in initialization, before the first step runs. There is an experimental flag ``--fast-parser`` which can dramatically reduce the initialization overhead, however as of this writing it has several limitations: From 1e150ab6ad9ebd329186d31ad010352e65605ceb Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 13 Sep 2022 18:16:01 +0200 Subject: [PATCH 25/28] conformance tests: report CWLTOOL_OPTIONS --- conformance-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conformance-test.sh b/conformance-test.sh index 016edfe22..d0ac50e1e 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -129,6 +129,7 @@ Conformance test of cwltool ${tool_ver} for CWL ${version} Commit: ${GIT_COMMIT} Python version: 3 Container: ${container} +Extra options: ${CWLTOOL_OPTIONS} EOM ) @@ -145,6 +146,7 @@ else EXCLUDE="" fi export CWLTOOL_OPTIONS +echo CWLTOOL_OPTIONS="${CWLTOOL_OPTIONS}" # shellcheck disable=SC2086 LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml ${EXCLUDE} \ RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \ From f3e14e1f11c93584ec4c1a44ea3dcdd8484bde35 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 13 Sep 2022 18:40:09 +0200 Subject: [PATCH 26/28] CI: include extra options in coverge classname --- conformance-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance-test.sh b/conformance-test.sh index d0ac50e1e..077808f4e 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -151,7 +151,7 @@ echo CWLTOOL_OPTIONS="${CWLTOOL_OPTIONS}" LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml ${EXCLUDE} \ RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \ ${DRAFT} \ - "--classname=py3_${container}" + "--classname=py3_${container}_$(echo ${CWLTOOL_OPTIONS} | tr "[:blank:]-" _)" # LC_ALL=C is to work around junit-xml ASCII only bug # capture return code of ./run_test.sh From 746033bb329b164f4e5007ddccfce1aff668415b Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 13 Sep 2022 13:17:52 -0400 Subject: [PATCH 27/28] Bump cwl-utils version requirement --- mypy-requirements.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 684853cbe..6d7fbd533 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,7 +1,7 @@ mypy==0.971 ruamel.yaml>=0.16.0,<0.17.22 schema-salad>=8.2.20211104054942,<9 -cwl-utils >=0.18 +cwl-utils >=0.19 types-requests types-setuptools types-psutil diff --git a/requirements.txt b/requirements.txt index 186c3b522..f485c88f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ pydot>=1.4.1 argcomplete>=1.12.0 pyparsing != 3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 pyparsing < 3;python_version<='3.6' # breaks --print-dot -cwl-utils>=0.18 +cwl-utils>=0.19 diff --git a/setup.py b/setup.py index 8650bdb6f..ee681a234 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "pyparsing < 3 ;python_version<='3.6'", # breaks --print-dot (pydot) "argcomplete", - "cwl-utils >= 0.18", + "cwl-utils >= 0.19", ], extras_require={ "deps": ["galaxy-tool-util >= 22.1.2, <23"], From 2aa44d0670bfad2edd0516fad78d168db81abd24 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 13 Sep 2022 19:38:33 +0200 Subject: [PATCH 28/28] conformance coverage: normalize paths --- conformance-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conformance-test.sh b/conformance-test.sh index 077808f4e..385bfd8f7 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -56,6 +56,7 @@ pip3 install -U setuptools wheel pip pip3 uninstall -y cwltool pip3 install -e . pip3 install codecov cwltest>=2.1 +root_folder=${PWD} pushd "${repo}-${spec_branch}" || exit 1 # shellcheck disable=SC2043 @@ -71,6 +72,7 @@ cat > "${COVERAGE_RC}" <