From 649afa6743dd4954352c8ed0e8124f63e3f8699d Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 19 Apr 2022 15:28:46 +0200 Subject: [PATCH 1/3] CWL: demonstrate use of conda suggestions Allows each step to use a different conda environment and the selection of conda packages that are not co-installable. --- simple_use_case/cwl/README.md | 7 +++++++ simple_use_case/cwl/compile_paper.cwl | 13 +++++++++---- simple_use_case/cwl/convert_msh_to_xdmf.cwl | 17 +++++++++++++---- simple_use_case/cwl/make_gmsh_mesh.cwl | 15 ++++++++++++--- simple_use_case/cwl/make_paraview_plot.cwl | 16 +++++++++++++--- simple_use_case/cwl/prepare_paper_macros.cwl | 3 +-- simple_use_case/cwl/run_dolfin.cwl | 14 +++++++++++--- 7 files changed, 66 insertions(+), 19 deletions(-) diff --git a/simple_use_case/cwl/README.md b/simple_use_case/cwl/README.md index 51f93ec3..064b2ecd 100644 --- a/simple_use_case/cwl/README.md +++ b/simple_use_case/cwl/README.md @@ -20,6 +20,13 @@ cwltool wf_run_use_case.cwl into your terminal. +Or you can use the conda dependency feature of the CWL reference runner to obtain +the dependencies dynamically: + +```sh +cwltool --beta-conda-dependencies wf_run_use_case.cwl +``` + Note that there exist tools to visualize, edit or create cwl workflows. For instance, you can visualize workflows contained in git repositories with [view.commonwl.org](https://view.commonwl.org/), or you can use the [Rabix Composer](https://github.com/rabix/composer) to compose workflows locally diff --git a/simple_use_case/cwl/compile_paper.cwl b/simple_use_case/cwl/compile_paper.cwl index 7915ce93..d5fe998e 100644 --- a/simple_use_case/cwl/compile_paper.cwl +++ b/simple_use_case/cwl/compile_paper.cwl @@ -2,16 +2,21 @@ cwlVersion: v1.0 class: CommandLineTool -doc: | - Create the paper as pdf, given the produced csv file -baseCommand: ["tectonic"] -arguments: ["$(inputs.texfile)"] +doc: Create the paper as pdf, given the produced csv file +baseCommand: tectonic +arguments: [ $(inputs.texfile) ] requirements: InitialWorkDirRequirement: listing: - $(inputs.csvfile) - $(inputs.texfile) - $(inputs.macros) +hints: + SoftwareRequirement: + packages: + tectonic: + version: [ 0.8.0=ha1fef3e_1, 0.8.0 ] + specs: [ https://anaconda.org/conda-forge/tectonic ] inputs: macros: type: File diff --git a/simple_use_case/cwl/convert_msh_to_xdmf.cwl b/simple_use_case/cwl/convert_msh_to_xdmf.cwl index 9145c828..68115949 100644 --- a/simple_use_case/cwl/convert_msh_to_xdmf.cwl +++ b/simple_use_case/cwl/convert_msh_to_xdmf.cwl @@ -2,10 +2,19 @@ cwlVersion: v1.0 class: CommandLineTool -doc: | - Convert the produced gmsh mesh to xdmf -baseCommand: ["meshio", "convert"] -arguments: ["$(inputs.inputmesh.path)", "mesh_converted.xdmf"] +doc: Convert the produced gmsh mesh to xdmf + +hints: + SoftwareRequirement: + packages: + meshio: + version: [ 5.0.5=pyhd8ed1ab_0, 5.0.5 ] + specs: + - https://anaconda.org/conda-forge/meshio + - https://doi.org/10.5281/zenodo.1173115 + +baseCommand: [ meshio, convert] +arguments: [ $(inputs.inputmesh.path), mesh_converted.xdmf ] inputs: inputmesh: type: File diff --git a/simple_use_case/cwl/make_gmsh_mesh.cwl b/simple_use_case/cwl/make_gmsh_mesh.cwl index 3882489e..6d8854ba 100644 --- a/simple_use_case/cwl/make_gmsh_mesh.cwl +++ b/simple_use_case/cwl/make_gmsh_mesh.cwl @@ -2,9 +2,18 @@ cwlVersion: v1.0 class: CommandLineTool -doc: | - Generate the computational mesh with gmsh -baseCommand: ["gmsh"] +doc: Generate the computational mesh with gmsh + +hints: + SoftwareRequirement: + packages: + gmsh: + version: [ 4.6.0=hd134328_0, 4.6.0 ] + specs: + - https://anaconda.org/conda-forge/gmsh + - https://identifiers.org/rrid/RRID:SCR_021226 + +baseCommand: gmsh arguments: ["-setnumber", "domain_size", "$(inputs.domain_size)", "-2", "$(inputs.geofile.path)", "-o", "mesh.msh"] diff --git a/simple_use_case/cwl/make_paraview_plot.cwl b/simple_use_case/cwl/make_paraview_plot.cwl index 048cc28c..7707a1f5 100644 --- a/simple_use_case/cwl/make_paraview_plot.cwl +++ b/simple_use_case/cwl/make_paraview_plot.cwl @@ -2,10 +2,20 @@ cwlVersion: v1.0 class: CommandLineTool -doc: | - Create plot-over-line data with paraview`s pvbatch -baseCommand: ["pvbatch"] +doc: Create plot-over-line data with paraview`s pvbatch +baseCommand: [pvbatch] arguments: ["$(inputs.script)", "$(inputs.pvdfile.path)", "plotoverline.csv"] + +hints: + SoftwareRequirement: + packages: + paraview: + version: [ 5.9.1=hfc1cbd4_3_egl, 5.9.1 ] + specs: + - https://anaconda.org/conda-forge/paraview + - https://identifiers.org/rrid/RRID:SCR_002516 + - https://bio.tools/paraview + requirements: InitialWorkDirRequirement: listing: diff --git a/simple_use_case/cwl/prepare_paper_macros.cwl b/simple_use_case/cwl/prepare_paper_macros.cwl index 89b16ed9..eb0226ca 100644 --- a/simple_use_case/cwl/prepare_paper_macros.cwl +++ b/simple_use_case/cwl/prepare_paper_macros.cwl @@ -2,8 +2,7 @@ cwlVersion: v1.0 class: CommandLineTool -doc: | - Update the macros of the paper with values from this run +doc: Update the macros of the paper with values from this run baseCommand: ["python3"] arguments: ["$(inputs.substitution_script)", "--domain-size", "$(inputs.domain_size)", diff --git a/simple_use_case/cwl/run_dolfin.cwl b/simple_use_case/cwl/run_dolfin.cwl index b0946914..e9b072e3 100644 --- a/simple_use_case/cwl/run_dolfin.cwl +++ b/simple_use_case/cwl/run_dolfin.cwl @@ -2,13 +2,21 @@ cwlVersion: v1.0 class: CommandLineTool -doc: | - Run the poisson solver in dolfin -baseCommand: ["python3"] +doc: Run the poisson solver in dolfin +baseCommand: python3 arguments: ["$(inputs.script)", "--mesh", "$(inputs.xdmfmeshfile.path)", "--degree", "2", "--output", "result.pvd"] stdout: output.txt +hints: + SoftwareRequirement: + packages: + fenics: + version: [ 2019.1.0=py39hf3d152e_26, 2019.1.0 ] + specs: + - https://anaconda.org/conda-forge/fenics + - https://bio.tools/fenics + requirements: InlineJavascriptRequirement: {} InitialWorkDirRequirement: From 2141e0ef1bfdd6a1fe6d05bb5e80789ef3f2e7bd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 20 Apr 2022 11:56:47 +0200 Subject: [PATCH 2/3] CWL: test auto conda feature --- .github/workflows/main.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5af37d9d..0129c72a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,6 +74,33 @@ jobs: retention-days: 1 if-no-files-found: error + run-cwl-auto-conda: + runs-on: ubuntu-latest + + steps: + - name: checkout-repository + uses: actions/checkout@v2 + + - name: install-basic-deps + uses: ./.github/actions/install-basic-deps + - name: prepare-conda + uses: conda-incubator/setup-miniconda@v2 + + - name: run-workflow + shell: bash -l {0} + run: | + sudo apt-get update && sudo apt-get install --no-install-recommends --yes python3-pip + python3 -m pip install cwltool[deps] + cd $GITHUB_WORKSPACE/simple_use_case/cwl + python3 -m cwltool --debug --beta-conda-dependencies wf_run_use_case.cwl + + - name: upload-paper-artifact + uses: actions/upload-artifact@v2 + with: + name: paper + path: ./simple_use_case/cwl/paper.pdf + retention-days: 1 + if-no-files-found: error run-nextflow: runs-on: ubuntu-latest From 9ada4d81527a2ff1167a0ebec2363000b0e095bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= <49902895+dglaeser@users.noreply.github.com> Date: Mon, 9 May 2022 16:46:02 +0200 Subject: [PATCH 3/3] [actions][cwl-auto-conda] add missing newline --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0129c72a..e4e5a9a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -83,6 +83,7 @@ jobs: - name: install-basic-deps uses: ./.github/actions/install-basic-deps + - name: prepare-conda uses: conda-incubator/setup-miniconda@v2