diff --git a/.github/actions/install-basic-deps/action.yml b/.github/actions/install-basic-deps/action.yml deleted file mode 100644 index e2fee2dc..00000000 --- a/.github/actions/install-basic-deps/action.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: install-basic-deps -runs: - using: "composite" - steps: - - run: sudo apt-get update && sudo apt-get install --no-install-recommends --yes libgomp1 libgl1-mesa-glx libglu1-mesa libegl1-mesa - shell: bash diff --git a/.github/workflows/cff.yml b/.github/workflows/cff.yml deleted file mode 100644 index 519cffd2..00000000 --- a/.github/workflows/cff.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: validate-citation-file - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - - workflow_dispatch: - -jobs: - validate-citation-cff: - runs-on: ubuntu-latest - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Validate CITATION.cff - uses: dieghernan/cff-validator@main diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 492bb3e7..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,306 +0,0 @@ -name: run-exemplary-workflow - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - run-pydoit: - runs-on: ubuntu-latest - - steps: - - name: checkout-repository - uses: actions/checkout@v2 - - - name: install-basic-deps - uses: ./.github/actions/install-basic-deps - - - name: setup-conda-environment - uses: conda-incubator/setup-miniconda@v2 - with: - environment-file: exemplary_workflow/source/envs/default_env.yaml - miniforge-version: latest - activate-environment: exemplary_workflow - - - name: run-workflow - shell: bash -l {0} - run: | - conda install --channel conda-forge doit=0.33.1 - cd $GITHUB_WORKSPACE/exemplary_workflow/pydoit - doit - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: ./exemplary_workflow/pydoit/paper.pdf - retention-days: 1 - if-no-files-found: error - - run-cwl: - runs-on: ubuntu-latest - - steps: - - name: checkout-repository - uses: actions/checkout@v2 - - - name: install-basic-deps - uses: ./.github/actions/install-basic-deps - - - name: setup-conda-environment - uses: conda-incubator/setup-miniconda@v2 - with: - environment-file: exemplary_workflow/source/envs/default_env.yaml - miniforge-version: latest - activate-environment: exemplary_workflow - - - name: run-workflow - shell: bash -l {0} - run: | - conda install --channel conda-forge cwltool - cd $GITHUB_WORKSPACE/exemplary_workflow/cwl - cwltool wf_run_exemplary_wf.cwl - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: ./exemplary_workflow/cwl/paper.pdf - retention-days: 1 - if-no-files-found: error - - run-nextflow: - runs-on: ubuntu-latest - - steps: - - name: checkout-repository - uses: actions/checkout@v2 - - - name: install-basic-deps - uses: ./.github/actions/install-basic-deps - - - name: setup-conda-environment - uses: conda-incubator/setup-miniconda@v2 - with: - miniforge-version: latest - activate-environment: exemplary_workflow - - - name: run-workflow - shell: bash -l {0} - run: | - conda install --channel bioconda nextflow=21.04.0 - cd $GITHUB_WORKSPACE/exemplary_workflow/nextflow - nextflow run exemplarywf.nf - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: ./exemplary_workflow/nextflow/paper.pdf - retention-days: 1 - if-no-files-found: error - - - run-snakemake: - runs-on: ubuntu-latest - - steps: - - name: checkout-repository - uses: actions/checkout@v2 - - - name: install-basic-deps - uses: ./.github/actions/install-basic-deps - - - name: setup-conda-environment - uses: conda-incubator/setup-miniconda@v2 - with: - miniforge-version: latest - activate-environment: exemplary_workflow - channels: conda-forge,bioconda - channel-priority: strict - - - name: run-workflow - shell: bash -l {0} - run: | - conda install snakemake - cd $GITHUB_WORKSPACE/exemplary_workflow/snakemake - snakemake --cores 1 --use-conda --conda-frontend conda ./paper.pdf - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: ./exemplary_workflow/snakemake/paper.pdf - retention-days: 1 - if-no-files-found: error - - run-kadistudio: - runs-on: ubuntu-latest - - steps: - - name: checkout-repository - uses: actions/checkout@v2 - - - name: install-basic-deps - uses: ./.github/actions/install-basic-deps - - - name: setup-conda-environment - uses: conda-incubator/setup-miniconda@v2 - with: - environment-file: exemplary_workflow/source/envs/default_env.yaml - miniforge-version: latest - activate-environment: exemplary_workflow - - - name: run-workflow - shell: bash -l -i {0} - run: | - bash $GITHUB_WORKSPACE/exemplary_workflow/kadistudio/install_components.sh - mkdir $GITHUB_WORKSPACE/run - process_engine run $GITHUB_WORKSPACE/exemplary_workflow/kadistudio/exemplary_workflow.flow -p $GITHUB_WORKSPACE/run - - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: ./exemplary_workflow/kadistudio/paper.pdf - retention-days: 1 - if-no-files-found: error - - run-aiida: - runs-on: ubuntu-latest - - services: - postgres: - image: postgres:10 - env: - POSTGRES_DB: test_aiida - POSTGRES_PASSWORD: '' - POSTGRES_HOST_AUTH_METHOD: trust - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - rabbitmq: - image: rabbitmq:3.8.14 - ports: - - 5672:5672 - slurm: - image: xenonmiddleware/slurm:17 - ports: - - 5001:22 - - steps: - - name: checkout-repository - uses: actions/checkout@v2 - - - name: install-basic-deps - uses: ./.github/actions/install-basic-deps - - - name: install-system-dependencies - run: | - sudo apt update - sudo apt install postgresql graphviz - - - name: setup-conda-environment - uses: conda-incubator/setup-miniconda@v2 - with: - environment-file: exemplary_workflow/source/envs/default_env.yaml - miniforge-version: latest - activate-environment: exemplary_workflow - - - name: install-aiida-shell - # the shell directive is necessary to properly activate the shell - # see https://github.com/marketplace/actions/setup-miniconda - shell: bash -l {0} - run: | - pip install "aiida-core>=2.0,<2.1" "aiida-shell==0.2.0" - - - name: setup-aiida-profile - shell: bash -l {0} - run: | - verdi quicksetup --non-interactive --profile default --email aiida@localhost --first-name Giuseppe --last-name Verdi --institution Khedivial --db-backend psql_dos - - - name: run-workflow - working-directory: exemplary_workflow/aiida - shell: bash -l {0} - run: - ./exemplary_workflow.py - - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: ./exemplary_workflow/aiida/paper.pdf - retention-days: 1 - if-no-files-found: error - - run-gwl: - runs-on: ubuntu-latest - - steps: - - name: Install Guix - id: install-guix - uses: PromyLOPh/guix-install-action@v1 - with: - channels: |- - (list (channel - (name 'guix) - (url "https://git.savannah.gnu.org/git/guix.git") - (branch "master") - (commit - "8e54584d4448d37ddf8ae995bb545a181ba2493c"))) - - - name: Install Guix Workflow Language - run: | - guix install gwl - echo "GUIX_EXTENSIONS_PATH=$HOME/.guix-profile/share/guix/extensions" >> $GITHUB_ENV - - - name: checkout-repository - uses: actions/checkout@v2 - - - name: run-workflow - run: | - cd exemplary_workflow/gwl - guix workflow run workflow.w - - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: exemplary_workflow/gwl/paper.pdf - - run-pyiron: - runs-on: ubuntu-latest - - steps: - - name: checkout-repository - uses: actions/checkout@v2 - - - name: install-basic-deps - uses: ./.github/actions/install-basic-deps - - - name: setup-conda-environment - uses: conda-incubator/setup-miniconda@v2 - with: - miniforge-version: latest - activate-environment: exemplary_workflow - channels: conda-forge - channel-priority: strict - - - name: run-workflow - shell: bash -l {0} - run: | - conda install conda_subprocess=0.0.4 pyiron_base=0.9.7 - cd $GITHUB_WORKSPACE/exemplary_workflow/pyiron - python workflow.py - - name: upload-paper-artifact - uses: actions/upload-artifact@v2 - with: - name: paper - path: ./exemplary_workflow/pyiron/paper.pdf - retention-days: 1 - if-no-files-found: error diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 00000000..3006ea06 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,74 @@ +name: Pipeline + +on: + push: + branches: [ main ] + pull_request: + +jobs: + aiida: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Conda config + run: echo -e "channels:\n - conda-forge\n" > .condarc + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: "3.12" + miniforge-version: latest + condarc-file: .condarc + environment-file: environment.yml + - name: Test + shell: bash -l {0} + run: | + sudo apt-get install -y $(cat apt.txt) + conda env create -n preprocessing -f source/envs/preprocessing.yaml -y + conda env create -n processing -f source/envs/processing.yaml -y + conda env create -n postprocessing -f source/envs/postprocessing.yaml -y + verdi presto --profile-name pwd + echo -e 'from aiida import load_profile\nload_profile()\n\nfrom python_workflow_definition.aiida import load_workflow_json\n\n\nif __name__ == "__main__":\n workgraph = load_workflow_json(file_name="workflow.json")\n workgraph.run()' > test_with_aiida.py + python test_with_aiida.py + + jobflow: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Conda config + run: echo -e "channels:\n - conda-forge\n" > .condarc + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: "3.12" + miniforge-version: latest + condarc-file: .condarc + environment-file: environment.yml + - name: Test + shell: bash -l {0} + run: | + sudo apt-get install -y $(cat apt.txt) + conda env create -n preprocessing -f source/envs/preprocessing.yaml -y + conda env create -n processing -f source/envs/processing.yaml -y + conda env create -n postprocessing -f source/envs/postprocessing.yaml -y + echo -e 'from jobflow.managers.local import run_locally\nfrom python_workflow_definition.jobflow import load_workflow_json\n\n\nif __name__ == "__main__":\n flow = load_workflow_json(file_name="workflow.json")\n print(run_locally(flow))' > test_with_jobflow.py + python test_with_jobflow.py + + pyiron: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Conda config + run: echo -e "channels:\n - conda-forge\n" > .condarc + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: "3.12" + miniforge-version: latest + condarc-file: .condarc + environment-file: environment.yml + - name: Test + shell: bash -l {0} + run: | + sudo apt-get install -y $(cat apt.txt) + conda env create -n preprocessing -f source/envs/preprocessing.yaml -y + conda env create -n processing -f source/envs/processing.yaml -y + conda env create -n postprocessing -f source/envs/postprocessing.yaml -y + echo -e 'from python_workflow_definition.pyiron_base import load_workflow_json\n\n\nif __name__ == "__main__":\n delayed_object_lst = load_workflow_json(file_name="workflow.json")\n print(delayed_object_lst[-1].pull())' > test_with_pyiron.py + python test_with_pyiron.py diff --git a/apt.txt b/apt.txt new file mode 100644 index 00000000..611b8e6c --- /dev/null +++ b/apt.txt @@ -0,0 +1,2 @@ +libgl1-mesa-glx +libegl1-mesa diff --git a/conf.py b/conf.py deleted file mode 100644 index afc3dc0b..00000000 --- a/conf.py +++ /dev/null @@ -1,77 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -import sphinx_rtd_theme - - -# -- Project information ----------------------------------------------------- - -project = 'NFDI4IngScientificWorkflowRequirements' -copyright = '2021, Jörg F. Unger' -author = 'Jörg F. Unger, Dennis Gläser, Philipp Diercks' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.mathjax', - 'sphinx.ext.githubpages', - 'sphinx_rtd_theme', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -source_suffix = { - '.rst': 'restructuredtext', -} - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['conda-env','conda-env', '_build', 'Thumbs.db', '.DS_Store', 'exemplary_workflow'] - - -# The master toctree document. -master_doc = 'index' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] diff --git a/docs/exemplarywf.rst b/docs/exemplarywf.rst deleted file mode 100644 index 455e05c0..00000000 --- a/docs/exemplarywf.rst +++ /dev/null @@ -1,78 +0,0 @@ - -.. _exemplarywf: - -Exemplary workflow -================== -As a minimal working example representative of workflows in computational science, the -poisson equation is solved using the finite element method for a set of predefined boundary conditions :math:`u_{\mathrm{D}}` -on a square domain :math:`\varOmega={(0, s)}^2, s\in\mathbb{R}^+`. - -.. math:: - -\Delta u &= f\,,\quad&&\mathrm{in}\;\varOmega\,,\\ - u &= u_{\mathrm{D}}\,,\quad&&\mathrm{on}\;\partial\varOmega\,. - -The domain size :math:`s` is defined as an input to the workflow, such that the workflow can -be run for different computational domains. -The final output of the workflow is a PDF document showing the solution :math:`u` over the -line :math:`L=\{(x, y)\in\varOmega\,\vert\,x=y\}` and the number of degrees of freedom used in the finite element model. -The (directed acyclic graph of the) workflow is shown below and consists of the following processes: - -.. |DAG| image:: ./../img/exemplary_wf_dag.png - :class: align-right - :width: 95% - :alt: exemplary_wf_dag - -1. Partition of the computational domain using `Gmsh `_, |DAG| - -.. code-block:: console - - gmsh -2 -setnumber domain_size 1.0 source/square.geo -o ./square.msh - - -2. Conversion of the file format (into one readable by `FEniCS `_) using `meshio `_, - -.. code-block:: console - - meshio convert ./square.msh ./square.xdmf - - -3. Solution of the poisson equation using `FEniCS `_, - -.. code-block:: console - - python source/poisson.py --mesh ./square.xdmf --degree 2 --outputfile ./poisson.pvd - - -4. Postprocessing using `ParaView `_, - -.. code-block:: console - - pvbatch source/postprocessing.py ./poisson.pvd ./plotoverline.csv - - -5. Preparation of macro definitions, - -.. code-block:: console - - python source/prepare_paper_macros.py --macro-template-file source/macros.tex.template \ - --plot-data-path ./plotoverline.csv --domain-size 1.0 --num-dofs num_dofs \ - --output-macro-file ./macros.tex - - -6. Generation of a PDF using `LaTeX `_, `Tectonic `_ respectively. - -.. code-block:: console - - cp source/paper.tex ./paper.tex - tectonic ./paper.tex - -It is important to note that the `source files `_ are -regarded as static inputs, i.e. changing them may break the workflow and thus -only the variable domain size should be exposed as an actual input parameter. -The number of degrees of freedom is written to stdout -by the FEniCS script `source/poisson.py` -and used here to showcase how one might deal with output of integer type (i.e. non-file output). -If the tool does not support this the number of degrees of freedom is written to a file. - -Details on the specific versions used for each software package can be found in the `conda environment specification file `_. -Exemplary implementations with various tools can be found `here `_. diff --git a/docs/fairness.rst b/docs/fairness.rst deleted file mode 100644 index cb58fa6a..00000000 --- a/docs/fairness.rst +++ /dev/null @@ -1,86 +0,0 @@ -.. _fairness: - -FAIRness of research workflows -============================== - -.. image:: ./../img/fairness.png - :width: 500 - :alt: TODO: caption - -As mentioned in the introduction, an overarching goal is to make research -software `FAIR `_. In this section, we -therefore want to discuss possible aspects of FAIRness associated with the code -and data that are used in or produced by scientific inquiry. A scientific workflow -may consist of a variety of different pieces of software as well as input data, -which makes it nontrivial to publish it in a comprehensive, reusable and reproducible -way. - -.. contents:: - -.. _transparency: - -Increasing transparency ------------------------- - -By publishing the data and software used in scientific investigations, researchers -give their peers the opportunity to retrace the steps that were necessary to arrive -at the conclusions drawn in a publication. This significantly increases the -transparency of their research and besides this, it allows others to reuse the data -and software in proceeding investigations. - - -.. _reusable_data: - -Reusable data -------------- - -Integration of externally published research data in a research project may be -cumbersome if the data is not well-documented or self-explanatory. To mitigate -this, data should be published in standard formats that are widely used in the -community to guarantee their seamless integration into other projects. Moreover, -rich metadata should be provided, which explains the contents of the data in -detail and in a machine-readable format. - -.. _reusable_software: - -Reusable software ------------------ - -Source code used to process or produce data should be published such -that other researchers can easily reconstruct how the data was generated or analyzed. -Besides a license which states terms and conditions of use, it is again important -to publish metadata alongside the code, which state the requirements of the software -on the compute environment, that is, compiler or interpreter versions, third-party libraries, -hardware requirements etc. - -*Ideally, this code expresses only project-specific tasks, reusing established -packages for generic functionality, whenever possible, instead of reimplementing -it. This reduces the risk of bugs and makes it easier for other researchers -to understand the code, who possibly are familiar with the syntax of the used -packages. If generic, possibly reusable functionality has to be implemented, we encourage -researchers to publish these parts separately in the form of a code repository -and/or software package for others to use independently of the project-specific -code. This, of course, requires competence in sustainable software development -or the help of research software engineers.* - -.. _reproducible workflows: - -Reproducible research workflows -------------------------------------- - -A scientific workflow may involve many different processes that are executed in -a particular order while exchanging data. With increasing number of processes, -retracing the workflow logic can become rather cumbersome. Using workflow tools -or languages to describe the logic and the flow of data provides the means to -communicate this in a more comprehensible way. Moreover, some tools provide -the possibility to visualize the workflow in a directed acyclic graph (DAG), -depicting inputs, outputs and metadata of the processes and how they are interconnected. -In addition to the graphical visualization of the provenance graph, the workflow tool -may also automatically track and record the full data provenance in a database. -Besides this, other researchers may easily extend or change the workflow by adding or exchanging processes. - -In order for the workflow to be reproducible, metadata must be provided that state -the requirements on the compute environment and all of its dependencies, such that -users can prepare their system for the workflow execution. Ideally, workflow tools -automatically track the involved dependencies and/or provide the means to encapsulate -the environment (e.g. using container solutions) such that it can be distributed. diff --git a/docs/requirements.rst b/docs/requirements.rst deleted file mode 100644 index 474aad2b..00000000 --- a/docs/requirements.rst +++ /dev/null @@ -1,226 +0,0 @@ -.. _requirements: - -Requirements on workflow tools -============================== - -In this section, we want to translate the aspects discussed in :ref:`fairness` -into possible capabilities that workflow tools should provide in order to -fulfill these aspects. - -.. contents:: - -.. _requirements_execution: - -Execution ---------- -The workflow tool should automatically execute the scientific workflow, i.e.\ the processes that make up the workflow should be executed in the correct order to satisfy dependencies between them. -To this end, a directed acyclic graph (DAG; dependency-graph) should be created and stored for later tracing and visualisation of specific instances of a workflow execution (see also :ref:`requirements_provenance`. -Moreover, the (possible) reuse of already computed results is an important feature with regard to the execution of the workflow, but also handled quite differently by the investigated tools. -Therefore, this is listed as a separate requirement (see :ref:`requirements_uptodateness`). -In this requirement, it is focused on how (easily) the execution of the workflow either locally (your computer) or on a remote machine (HPC cluster or cloud) can be integrated. -Note, that this relates to the configuration of remote machines and e.g. the ressource manager on a HPC cluster or the cloud computing service with the workflow tool rather than the actual execution, since for this a suitable compute environment needs to be instantiated which is covered in :ref:`requirements_compute_environment`. -Ideally, the workflow can be executed anywhere without changing the workflow definition script apart from small changes (one-liner) related to the workflow tool configuration file. - -Evaluation criteria: - -1. The workflow system supports the execution of the workflow on the local system. -2. The workflow system supports the execution of the workflow on the local system via a batch system. -3. The workflow system supports the execution of the workflow via a batch system on the local or a remote system. - -.. _requirements_monitor: - -Monitoring ----------- -Depending on the application, the execution of scientific workflows can be very time-consuming. This can be caused by compute-intensive processes, -such as e.g. numerical simulations, or by a large number of short processes that are executed many times. In both cases, it can be very helpful to -be able to query the state of the execution, that is, which processes have been finished, which processes are currently being processed, and which -are still pending. A trivial way of such monitoring would be, for instance, when the workflow is started in a terminal which is kept open to inspect -the output written by the workflow system and the running processes. However, ideally, the workflow system allows for submission of the workflow in -the form of a process running in the background, while still providing means to monitor the state of the execution. - -Evaluation criteria: - -1. Only way to monitor the workflow is to watch the console output -2. The workflow system provides a way to query the execution status at any time - -.. _requirements_provenance: - -Data provenance ---------------- -The data provenance graph contains, for a particular execution of the workflow, which data and processes participated in the generation of a particular -piece of data. Thus, this is closely related to the workflow itself, which can be thought of as a template for how that data generation should take place. -However, a concrete realization of the workflow must contain information on the exact input data and parameters that were used, possibly along with meta -information on the person that executed the workflow, the compute ressources used and the time it took to finish. Collection of all relevant information, -its storage in machine-readable formats and subsequent publication alongside the data can be very useful for future researchers in order to understand -how exactly the data was produced. Ideally, the workflow system has the means to automatically collect this information upon workflow execution. - -Evaluation criteria: - -1. The workflow system provides no means to export relevant information from a particular execution -2. Upon workflow execution, the tool writes metadata files alongside the results, overwriting them upon re-execution -3. Produced data is stored in a database, allowing to uniquely associate produced data with particular workflow instantiations - - -.. _requirements_compute_environment: - -Compute environment -------------------- -In order to guarantee interoperability and reproducibility of scientific workflows, the workflows need to be executable by others. -Here, the reinstantiation of the compute environment (installation of libraries or source code) poses the main challenge. -Therefore, it is of great use that the workflow tool is able to automatically deploy the software stack (on a per workflow or per process basis) by means of a package manager (e.g. conda) or that running processes in a container (e.g. docker, singularity, etc.) is integrated in the tool. - -Evaluation criteria: - -1. The automatic instantiation of the compute environment is not intended. -2. The workflow system allows the automatic instantiation of the compute environment on a per workflow basis. -3. The workflow system allows the automatic instantiation of the compute environment on a per process basis. - -.. _requirements_uptodateness: - -Up-to-dateness --------------- -There are different areas for the application of workflows. On the one hand, -people might use a workflow to define a single piece of reproducible code -that when executed, always returns the same result. Based on that they might -start a large quantity of different jobs and use the workflow system to -perform this task. Another area of application is the constant development -within the workflow (e.g. exchanging processes, varying parameter or even -modifying the source code of a process) until a satisfactory result is -obtained. The two scenarios require a slightly different behavior of the -workflow system. In the first scenario, all runs should be kept in the data -provenance graph with a documentation of how each result instance has been -obtained (e.g. by always documenting the codes, parameters, and processes). -If identical runs (identical inputs and processes should result in the same -output) are detected, a recomputation should be avoided and the original -output should be linked in the data provenance graph. The benefit of this -behavior certainly depends on the ratio between the computation time for a -single process compared to the overhead to query the data base. - -However, when changing the processes (e.g. coding a new time integration -scheme, a new constitutive model), the workflow system should rather behave -like a built system (such as make) - only recomputing the steps that are -changed or that depend on these changes. In particular for complex problems, -this allows to work with complex dependencies without manually triggering -computations and results in automatically recomputing only the relevant parts -. An example is a paper with multiple figures that each is a result of -complex simulations that in itself depend on a set of general modules that -are developed in the paper. The "erroneus" runs are usually not interesting -and should be overwritten. - -How this is handled varies between the tools. Some always recompute the -complete workflow marked in the matrix by an **R**\ ecompute, others allow -to create a new entry in the data provenance graph and link the previous -result (without the need to recompute already existing results) marked in the -matrix as **L**\ ink. Finally, make-like tools recreate only the parts -that are not up-to-date labeled as **U**\ pdate. Note that the latter -usually reduces the overhead to store multiple instances of the workflow, but -at the same time also prevents - without additional effort (e.g. when -executing in different folders) computing multiple instances of the same -workflow. - - -.. _requirements_gui: - -Graphical user interface ------------------------- -Independent of a particular execution of the workflow, the workflow system may provide facilities to visualize the graph of the workflow, indicating the -mutual dependencies of the individual processes and the direction of the flow of data. One can think of this graph as the template for the data provenance -graph. This visualization can help in conveying the logic behind a particular workflow, making it easier for other researchers to understand and possibly -incorporate it into their own research. The latter requires that the workflow system is able to handle hierarchical workflows, i.e. it needs to support -sub-workflows as processes inside another workflow. Beyond a mere visualization, a graphical user interface may allow for visually connecting different -workflows into a new one by means of drag & drop. An example for this is the [Rabix Composer](https://github.com/rabix/composer), which allows for the composition of workflows -written in CWL. - -Evaluation criteria: - -1. The workflow system provides no means to visualize the workflow -2. The workflow system or third-party tools allow to visualize the workflow definition -3. The workflow system or third-party tools provide a graphical user interface that enables users to graphically create workflows - -.. _requirements_hierarchical: - -Hierarchical composition of workflows -------------------------------------- -A workflow consists of a mapping between a set of inputs (could be empty) and -a set of outputs, whereas in between a number of sequential processes are -performed. Connecting the output of one workflow to the input of another -workflow results in a new, longer workflow. This is particularly relevant in -situations, where multiple people share a common set of procedures (e.g. -common pre- and postprocessing routines). In this case, copying the -preprocessing workflow into another one is certainly always possible, but -does not allow to jointly perform modifications and work with different -versions. If the workflow system supports a hierarchical embedding of one -workflow into another one, the property is labeled as + (otherwise -). This -also requries to define separate compute environments for each sub-workflow -(e.g. docker/singularity or conda), because each sub-workflow might use -different tools or even the same tools but with different versions (e.g. -python2 vs. python3), so executing all sub-workflows in the same environment -might not be possible. - -.. _requirements_interfaces: - -Process interfaces ------------------- -Each process in a workflow has some input and output data. -In a traditional file based pipeline the output of one process is input to the other. -However, it is often more convenient to pass non-file output (e.g. float or integer values) directly from one process to the other without the creation of intermediate files. -In this case, it is desirable that the workflow tool is able to check for the validity of the data (e.g. the correct data type) to be processed. -Furthermore, this clearly defines the interface for a process and which input values may be changed. -This way, a third person is able to understand how to work with, adapt and extend the workflow/process. -In contrast, in a file based pipeline this is usually not the case, since a dependency in form of a file does not give information about the type of data contained in that file. - -Evaluation criteria: - -1. The workflow system is purely file-based and does not define interface formats. -2. The workflow system has a file and non-file based interface, where the non-file based inputs are well defined. -3. The workflow system has a file and non-file based interface, where both the file and non-file based inputs are well defined. - -.. _requirements_platform: - -Platform for publishing and sharing workflows ---------------------------------------------- -The benefit of a workflow system is already significant when using it for -individual research such as the development of my paper or reproducing the -paper someone else has written, when their data processing pipeline is fully -reproducible and documented and published with the publication. However, the -benefit can be even more increased if people are able to jointly work on -(sub-)workflows together. In particular, when a hierarchical workflow system -is used. Even though workflows can easily be shared together with the work (e -.g. in a repository), it might be beneficial to provide a platform that -allows to publish documented workflows with a search and versioning -functionality. This feature is not part of the requirement matrix to compare -the different tools, but we consider a documentation of these platforms (if -existing) in the subsequent section important source of information as a good -starting point for further research (exchange). - -.. _requirements_evaluation: - -Evaluation ----------- - -.. https://www.unicode-search.net/unicode-namesearch.pl?term=CIRCLE -.. ● BLACK CIRCLE -.. ○ WHITE CIRCLE -.. 🔴 large red circle to indicate important requirement for user story - -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| Tool | Execution | Monitoring | Provenance | Compute Environment | GUI | Composition | Process Interfaces | Up-to-dateness | Ease-of-first-use | -+============================+===========+============+============+=====================+=====+=============+====================+================+===================+ -| AiiDA | ●●● | ●● | ●●● | ●○○ | ●●○ | ●●○ | ●●○ | L | ●○○ | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| CWL | ●●○ | ●● | ●●○ | ●●● | ●●● | ●●● | ●●● | R | ●●○ | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| Doit | ●○○ | ●○ | ●○○ | ●○○ | ●○○ | ●●○ | ●○○ | U | ●●● | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| Nextflow | ●●○ | ●○ | ●●○ | ●●● | ●●○ | ●●● | ●○○ | L | ●●● | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| Snakemake | ●●○ | ●○ | ●●○ | ●●● | ●●○ | ●●● | ●○○ | U | ●●● | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| User Story | Execution | Monitoring | Provenance | Compute Environment | GUI | Composition | Process Interfaces | Up-to-dateness | Ease-of-first-use | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| Paper | | | | 🔴 | | | | 🔴 | 🔴 | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| Joint research | | | 🔴 | 🔴 | | 🔴 | 🔴 | 🔴 | | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ -| Complex hier. computations | 🔴 | 🔴 | 🔴 | 🔴 | | 🔴 | | | | -+----------------------------+-----------+------------+------------+---------------------+-----+-------------+--------------------+----------------+-------------------+ diff --git a/docs/tools.rst b/docs/tools.rst deleted file mode 100644 index 407f658f..00000000 --- a/docs/tools.rst +++ /dev/null @@ -1,111 +0,0 @@ -.. _developers: - -List of tools -============= -This is a list of preselected tools based on possible requirements at the start of -the project. - -.. contents:: - -AiiDA/AiiDAlab --------------- -* `AiiDA Website `_ -* `AiiDALab Website `_ -* `Main Paper `_ (AiiDA >= 1.0) with (so it seems) main authors - Sebastiaan P. Huber and Spyros Zoupanos (no longer active member of the AiiDA team). Correspondence and requests for materials should be addressed to S.P.H. or Giovanni Pizzi. -* `AiiDA mailing list `_ for general questions about AiiDA. -* `Github Issue Tracker `_ for bug reports. - -Argo ----- -* `Argo website `_ - - -Common Workflow Language ------------------------- -* `CWL website `_ -* `Discourse group `_ as recommended place to ask questions about CWL. -* `CWL paper `_ -* Besides the discourse group one could get in contact via `Gitter `_, `Twitter `_ or `google mailinglist `_ - - -Dlite ------ -* `Dlite website `_ - - -Gnu guix --------- -* `Gnu guix website `_ -* `guix mailing list `_ - - -Kadi4Mat --------- -* `Kadi4Mat website `_ - - -Kepler ------- -* `Kepler website `_ - - -Nextflow --------- -* `Nextflow website `_ -* `Nextflow docs `_ - - -Pegasus -------- -* `Pegasus website `_ -* `Pegasus Contact `_ via Slack or mailing lists - - -Pydoit ------- -* `Pydoit website `_ -* created and maintained by Eduardo Schettino -* `Discussion Group `_ -* Info from the discussion group: Unless you are looking for paid support, do **not** send private emails to the project maintainer. - - -Pyiron ------- -* `Pyiron website `_ -* Contact form on the `team website `_. Find also link to email of e.g. - lead developer or project lead here. - - -Reana ------ -* `Reana website `_ - - -Snakemake ---------- -* `Snakemake website `_ -* `Credits `_ shows list of - development team members and lead Johannes Köster -* *rolling* `paper `_ which is updated regularly. - - -Swift ------ -* `Swift website `_ - - -Taverna -------- -* `Tavernas website `_ - - -Workflow hub ------------- -* `Workflow hub website `_ - - -Related lists -------------- -* `Awesome materials informatics `_ -* `Awesome pipeline `_ diff --git a/docs/userstories.rst b/docs/userstories.rst deleted file mode 100644 index 8b25b5e4..00000000 --- a/docs/userstories.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. _userstories: - -User stories -============ - "A user story is an informal, general explanation of a software feature written from the perspective of the end user. - Its purpose is to articulate how a software feature will provide value to the customer." - - "As a [persona], I [want to], [so that]." - (Source: `Atlassian `_) - -The concept of a user story as a component of agile software development is used here to describe a certain application scenario. -We aim to deduce :ref:`requirements` from challenges posed in the respective user stories below. - -.. _user_story_1: - -Reproducible paper ------------------- - *As a researcher, I want to share my paper, such that others are able to reproduce the results.* - -In this user story, the main objective is to guarantee reproducibility of the results described in a scientific publication. -To do so, other scientists need to be able to comprehend and rerun each process involved in the research workflow (numerical analysis, postprocessing, etc.). - -There are several challenges in meeting the goal above. -First of all, the code has to be published. -This can happen in the form of a tarball (e.g. via `Zenodo `_), or by employing version control on the code base and making the repository publicly accessible (e.g. via `GitHub `_). -This makes it possible to refer to a particular version of the code used at the time of publication. -As research software is also understood as research data, appropriate metadata regarding the code and dependencies on third-party libraries should be provided. -The metadata should contain all information necessary for peers to reinstantiate the compute environment to make the workflow usable. -However, since software becomes out of date rather quickly this is a minimum requirement and it is preferred to build containers that package up all required pieces of software in a way that is portable, reproducible and ensures machine-independent execution. -Finally, the fully automated implementation of the entire workflow is required to avoid any manual steps. -Ideally, the whole paper can be reproduced by running a single command and the progress of the execution is monitored by the workflow tool. - - -.. _user_story_2: - -Research group collaboration ----------------------------- - *As part of a research group, I want to be able to interconnect and reuse components of several different workflows so that everyone may benefit from their colleagues' work.* - -Similar to the first user story the output of the workflow could be a scientific paper. However with this example, interdisciplinary workflows are addressed and the reusability of single components/modules is essential. Each process in the workflow may require a different expertise and hence modularity and a common framework is a necessary requirement for an efficient collaboration and reuse of other tools. Moreover, joint software development and the development of the scientific workflow itself play an important role. - -Herein, the following challenges might occur. -First, the workflow may consist of heterogeneous models of different complexity, such as large computations requiring HPC potentially including many dependent substeps to be executed, preprocessing of experimental data or joint analysis in postprocessing step - all processes potentially running on different machines. -These models then need to be embedded into a common framework. -Without a common framework/interface the exchange of data between processes is challenging and requires manual adjustments. -Second, during joint development of the workflow, it needs to be executable independent of the local machine. -Therefore, the workflow tool must provide means to control (automatically install) the compute environment for specific processes of the workflow. -This greatly enhances the reusability of a process and/or modules (chain of processes) of the workflow and guarantees the machine-independent execution. -Another challenge are high computational costs. -Here, the seamless integration of HPC systems is of high value. -Moreover, for complex workflows containing computationally expensive processes, caching of results becomes relevant. -Upon changes in the workflow only parts of the workflow which are not up-to-date are rerun. -This saves computation time and may lead to a significant speed up in the time needed for development of the workflow. -Finally, a hierarchically embedding of a versioned published workflow as a sub-workflow in another workflow is of great benefit - thus improvements within this sub-workflow can be easily integrated. - -.. _user_story_3: - - -Complex hierarchical computations ---------------------------------- - *As a materials scientist, I want to be able to automate and manage complex workflows so I can keep track of all associated data.* - -In cases where screening or parameter sweeps are required, involving thousands of simulations, running these manually one by one is not feasible. -In addition, these workflows are often complex, i.e. the output of one process is the input to a subsequent one - potentially with many levels of dependencies. - -Moreover, besides the automation of running the calculations, the inputs and outputs need to be managed. -Not only the data and calculations should be stored to achieve reproducibility, but also the causal relationships between them, i.e. the full data provenance. - -Given the large amount of data (inputs/outputs), manually keeping track of the full provenance becomes infeasible. -Therefore, the workflow tool must automatically track and record inputs, outputs and metadata of all processes in a database and allow to easily search for the relavant (meta-)data. -Furthermore, fast queries of the database and automatic generation of the provenance graph are required features of the workflow tool. -Due to the large computational effort the seamless integration of HPC systems is as well vital for this use case. diff --git a/dodo.py b/dodo.py deleted file mode 100644 index e460ef47..00000000 --- a/dodo.py +++ /dev/null @@ -1,13 +0,0 @@ -import pathlib - - -def task_website(): - deps = ["index.rst", "conf.py"] - docs = pathlib.Path("docs") - for f in docs.glob("*.rst"): - deps.append(str(f)) - return { - "file_dep": deps, - "actions": ["sphinx-build . website"], - "verbosity": 2, - } diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..749859dd --- /dev/null +++ b/environment.yml @@ -0,0 +1,7 @@ +channels: +- conda-forge +dependencies: +- python =3.12 +- conda_subprocess =0.0.6 +- pip: + - python-workflow-definition==0.0.1 diff --git a/exemplary_workflow/README.md b/exemplary_workflow/README.md deleted file mode 100644 index bcb0fb0d..00000000 --- a/exemplary_workflow/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Exemplary workflow -As a minimal working example representative of workflows in computational science, -the poisson equation is solved using the finite element method. -The workflow consists of the following processes: - -1. Partition of the computational domain using [Gmsh](http://gmsh.info/), -2. Conversion of the file format (into one readable by [FEniCS](https://fenicsproject.org/)) using [meshio](https://github.com/nschloe/meshio), -3. Solution of the poisson equation using [FEniCS](https://fenicsproject.org/), -4. Postprocessing using [ParaView](https://www.paraview.org/), -5. Preparation of macro definitions, -6. Generation of a PDF using [LaTeX](https://www.latex-project.org/), [Tectonic](https://tectonic-typesetting.github.io/en-US/) respectively. - -A more extensive description can be found in the [documentation](https://nfdi4ingscientificworkflowrequirements.readthedocs.io/en/latest/docs/exemplarywf.html). - -## Compute environment -Details about how the compute environment is built (using [conda](https://docs.conda.io/en/latest/)) can be found in the respective sub-directory for each tool. - -## Headless operation -The ParaView version used in the examples (see the [conda environment specification file](https://github.com/BAMresearch/NFDI4IngScientificWorkflowRequirements/blob/main/exemplary_workflow/source/envs/default_env.yaml)) is linked against EGL to also support offscreen rendering. -Usually, these libraries exist in case of a desktop pc where an actual screen or monitor is available. -However, for headless operation, i.e. when executing the exemplary workflow in a container, it is necessary to install aforementioned libraries since these are not installed automatically as a dependency by conda. -We refer to the installation of the basic dependencies in our [github action](https://github.com/BAMresearch/NFDI4IngScientificWorkflowRequirements/blob/main/.github/actions/install-basic-deps/action.yml) to give an example. -For more information about offscreen rendering with ParaView go to the [ParaView documentation](https://kitware.github.io/paraview-docs/latest/cxx/Offscreen.html). diff --git a/exemplary_workflow/aiida/.gitignore b/exemplary_workflow/aiida/.gitignore deleted file mode 100644 index 7c58f1ae..00000000 --- a/exemplary_workflow/aiida/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -outfile* -*.pvd -*.vtu -*.msh -*.pdf diff --git a/exemplary_workflow/aiida/README.md b/exemplary_workflow/aiida/README.md deleted file mode 100644 index abe6ffb2..00000000 --- a/exemplary_workflow/aiida/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# AiiDA -This directory contains an implementation of the exemplary workflow with [AiiDA](https://www.aiida.net/). - -## Implementation -Since the implementation of workflows in AiiDA is quite different from the other file -based workflow managers (like e.g. snakemake or nextflow), we briefly comment on the different options -and design choices in AiiDA. - -### Calculation functions -According to the [documentation](https://aiida.readthedocs.io/projects/aiida-core/en/latest/topics/calculations/concepts.html#calculation-functions): -> The calcfunction in AiiDA is a function decorator that transforms a regular python function in a calculation process, which automatically stores the provenance of its output in the provenance graph when executed. - -Typically `calcfunction`s are used for short running processes to be run on the local machine, like preprocessing and postprocessing steps. -One could think of a workaround, using `os.subprocess` inside a `calcfunction` to run the processes of the exemplary workflow. -However, `calcfunction`s are not intended to be used to run external codes and the use of `os.subprocess` is discouraged since in this case the provenance cannot be properly captured by AiiDA. - -### Calculation jobs -> ... not all computations are well suited to be implemented as a python function, but rather are implemented as a separate code, external to AiiDA. To interface an external code with the engine of AiiDA, the CalcJob process class was introduced - -The `CalcJob` is designed to run a `Code` on *any* computer through AiiDA. -While this is very powerful, apart from installing the `Code` on the other computer it is necessary to setup the `code` with AiiDA and [write a plugin](https://aiida.readthedocs.io/projects/aiida-core/en/latest/howto/plugin_codes.html) which instructs AiiDA how to run the external `Code`. -For long running processes (computationally expensive tasks) this is worth the while, but for simple -shell commands the effort is too high. - -### AiiDA shell plugin -The [AiiDA shell plugin](https://github.com/sphuber/aiida-shell) was developed to make it easier to run simple shell commands with AiiDA. -This way any command line tool (external code) installed on the *computer* can be run without the need to write a plugin. -Moreover, the `ShellJob` inherits from `CalcJob` and thus it is possible to run commands on remote computers. -Instructions on how to setup a remote computer can be found in this [how-to-guide](https://aiida.readthedocs.io/projects/aiida-core/en/latest/howto/run_codes.html#how-to-set-up-a-computer). - -## Installation -Please follow the instructions in the [documentation](https://aiida.readthedocs.io/projects/aiida-core/en/latest/) -to make yourself familiar with the installation process. -It is recommended to use the system-wide installation method, where you first install prerequisite -services using a package manager (e. g. on Ubuntu) -```sh -sudo apt install \ - git python3-dev python3-pip \ - postgresql postgresql-server-dev-all postgresql-client rabbitmq-server -``` -Next, we prepare a conda environment with all the software required to run the exemplary workflow. -```sh -conda env create --name aiida_env --file ../source/envs/default_env.yaml -conda activate aiida_env -``` -Make sure that the python version is greater than 3.8, since this is required by the `aiida-shell` plugin. -Next we install the `aiida-shell` plugin, which will automatically install AiiDA as a dependency. -Make sure that your conda environment is activated as above and run the following commands. -```sh -pip install aiida-shell==0.2.0 -``` -Finally, run -```sh -verdi quicksetup -``` -to setup a profile and see if everything was installed correctly by running -```sh -verdi status -``` - -## Running the exemplary workflow -If you are using `conda`, activate your environment. -``` -conda activate aiida_env -``` -Make the workflow script executable (`chmod +x ./exemplary_workflow.py`) and run it with -``` -./exemplary_workflow.py -``` -By default all `ShellJob`s are run on the `localhost`. -Some useful commands to inspect the status of the processes run and their results stored in the database are listed below. -``` -verdi process list -a # lists all processes -verdi process show # show info about process -verdi process report # log messages if something went wrong -verdi node show # show info about node -verdi node graph generate # generate provenance graph for node -``` -The provenance graph can be created in `.png` format with the command -``` -verdi node graph generate --output-format png -``` -and is shown below. - -
diff --git a/exemplary_workflow/aiida/exemplary_workflow.py b/exemplary_workflow/aiida/exemplary_workflow.py deleted file mode 100755 index a66c743b..00000000 --- a/exemplary_workflow/aiida/exemplary_workflow.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env runaiida -from aiida.engine import calcfunction -from aiida.orm import Float, Int -from aiida_shell import launch_shell_job - -# ### generate mesh with gmsh -gmsh_results, gmsh_node = launch_shell_job( - "gmsh", - arguments=[ - "-2", - "-setnumber", - "domain_size", - "2.0", - "{geometry}", - "-o", - "mesh.msh", - ], - nodes={"geometry": "../source/unit_square.geo"}, - outputs=["mesh.msh"], -) - -# ### convert mesh from msh to xdmf format -meshio_results, meshio_node = launch_shell_job( - "meshio", - arguments=["convert", "{mesh}", "mesh.xdmf"], - nodes={"mesh": gmsh_results["mesh_msh"]}, - filenames={"mesh": "mesh.msh"}, - outputs=["*.xdmf", "*.h5"], -) - -# ### solution of the poisson problem with fenics -fenics_results, fenics_node = launch_shell_job( - "python", - arguments=[ - "{script}", - "--mesh", - "{mesh}", - "--degree", - "2", - "--outputfile", - "poisson.pvd", - ], - nodes={ - "script": "../source/poisson.py", - "mesh": meshio_results["mesh_xdmf"], - "mesh_h5": meshio_results["mesh_h5"], - }, - filenames={"mesh": "mesh.xdmf", "mesh_h5": "mesh.h5"}, - outputs=["poisson.pvd", "poisson000000.vtu"], -) - -# ### postprocessing of the fenics job -paraview_results, paraview_node = launch_shell_job( - "pvbatch", - arguments=["{script}", "{pvdfile}", "plotoverline.csv"], - nodes={ - "script": "../source/postprocessing.py", - "pvdfile": fenics_results["poisson_pvd"], - "vtufile": fenics_results["poisson000000_vtu"], - }, - filenames={"pvdfile": "poisson.pvd", "vtufile": "poisson000000.vtu"}, - outputs=["plotoverline.csv"], -) - - -@calcfunction -def get_domain_size(gmsh_stdout): - string = gmsh_stdout.get_content().split("Used domain size:")[1] - size = float(string.split("Used mesh size")[0]) - return Float(size) - - -@calcfunction -def get_num_dofs(fenics_stdout): - stdout = fenics_stdout.get_content() - ndofs = stdout.split("Number of dofs used:")[1] - return Int("".join(ndofs.split())) - - -# ### prepare latex macros -macros, macros_node = launch_shell_job( - "python", - arguments=[ - "{script}", - "--macro-template-file", - "{template}", - "--plot-data-path", - "{csvfile}", - "--domain-size", - "{domain_size}", - "--num-dofs", - "{num_dofs}", - "--output-macro-file", - "macros.tex", - ], - nodes={ - "script": "../source/prepare_paper_macros.py", - "template": "../source/macros.tex.template", - "csvfile": paraview_results["plotoverline_csv"], - "domain_size": get_domain_size(gmsh_results["stdout"]), - "num_dofs": get_num_dofs(fenics_results["stdout"]), - }, - filenames={"csvfile": "plotoverline.csv"}, - outputs=["macros.tex"], -) - -# ### compile paper -paper, paper_node = launch_shell_job( - "tectonic", - arguments=["{texfile}"], - nodes={ - "texfile": "../source/paper.tex", - "macros": macros["macros_tex"], - "csvfile": paraview_results["plotoverline_csv"], - }, - filenames={ - "texfile": "paper.tex", - "macros": "macros.tex", - "csvfile": "plotoverline.csv", - }, - outputs=["paper.pdf"], -) - -# ### extract final PDF from database -with open("paper.pdf", "wb") as handle: - handle.write(paper["paper_pdf"].get_object_content(path="./paper.pdf", mode="rb")) diff --git a/exemplary_workflow/cwl/.gitignore b/exemplary_workflow/cwl/.gitignore deleted file mode 100644 index 2c78f67c..00000000 --- a/exemplary_workflow/cwl/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -# environment folder as mentioned in readme -simpleusecase - -# potential artifacts of workflow execution -*.pdf -*.csv -*.vtu -*.pvd -*.msh diff --git a/exemplary_workflow/cwl/README.md b/exemplary_workflow/cwl/README.md deleted file mode 100644 index 89ff08a9..00000000 --- a/exemplary_workflow/cwl/README.md +++ /dev/null @@ -1,36 +0,0 @@ -Workflow implemetation with CWL -=============================== - -The implementation with CWL requires that all the dependencies are provided by the environment. -If you have [`miniconda`](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) -installed, you can simply create a respective environment by typing - -```sh -conda env create --file default_env.yml --prefix ./exemplarywf -conda activate ./exemplarywf -``` - -Note that by specifying the `--prefix` option you can simply remove all downloaded packages afterwards -by removing the folder given to `prefix`. To execute the workflow after activating the environment, -simply type - -```sh -cwltool wf_run_exemplary_wf.cwl -``` - -into your terminal. - -Or you can use the conda dependency feature of the CWL reference runner to obtain -the dependencies dynamically: - -```sh -cwltool --beta-conda-dependencies wf_run_exemplary_wf.cwl -``` - -Note that there exist tools to visualize, edit or create cwl workflows. For instance, you can -visualize workflows contained in git repositories with [view.commonwl.org](https://view.commonwl.org/), -or you can use the [Rabix Composer](https://github.com/rabix/composer) to compose workflows locally -on your machine. - -Example: ![Workflow Diagram](workflow.svg) -[source](https://view.commonwl.org/workflows/github.com/BAMresearch/NFDI4IngScientificWorkflowRequirements/blob/main/exemplary_workflow/cwl/wf_run_exemplary_wf.cwl) diff --git a/exemplary_workflow/cwl/compile_paper.cwl b/exemplary_workflow/cwl/compile_paper.cwl deleted file mode 100644 index 58f9e04a..00000000 --- a/exemplary_workflow/cwl/compile_paper.cwl +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: CommandLineTool - -doc: Create the paper as pdf, given the produced csv file - -baseCommand: tectonic - -arguments: [ $(inputs.texfile) ] - -requirements: - InitialWorkDirRequirement: - listing: - - $(inputs.csvfile) - - $(inputs.texfile) - - $(inputs.macros) - -hints: - DockerRequirement: - dockerPull: dxjoke/tectonic-docker - SoftwareRequirement: - packages: - tectonic: - version: [ 0.8.0=ha1fef3e_1, 0.8.0 ] - specs: [ https://anaconda.org/conda-forge/tectonic ] - -inputs: - macros: - type: File - csvfile: - type: File - format: iana:text/csv - texfile: - type: File - default: - class: File - location: ../source/paper.tex - -outputs: - pdf: - type: File - format: iana:application/pdf - outputBinding: - glob: "*.pdf" - -$namespaces: - iana: https://www.iana.org/assignments/media-types/ diff --git a/exemplary_workflow/cwl/convert_msh_to_xdmf.cwl b/exemplary_workflow/cwl/convert_msh_to_xdmf.cwl deleted file mode 100644 index 346b56f9..00000000 --- a/exemplary_workflow/cwl/convert_msh_to_xdmf.cwl +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 -class: CommandLineTool -doc: Convert the produced gmsh mesh to xdmf - -hints: - SoftwareRequirement: - packages: - meshio: - version: [ 5.0.5=pyhd8ed1ab_0, 5.0.5 ] - specs: - - https://anaconda.org/conda-forge/meshio - - https://doi.org/10.5281/zenodo.1173115 - -baseCommand: [ meshio, convert ] - -arguments: [ $(inputs.inputmesh.path), mesh_converted.xdmf ] - -inputs: - inputmesh: - type: File - -outputs: - outputmesh: - type: File - outputBinding: - glob: mesh_converted.xdmf - outputmeshdata: - type: File - outputBinding: - glob: mesh_converted.h5 diff --git a/exemplary_workflow/cwl/default_env.yml b/exemplary_workflow/cwl/default_env.yml deleted file mode 100644 index 2684de00..00000000 --- a/exemplary_workflow/cwl/default_env.yml +++ /dev/null @@ -1,9 +0,0 @@ -channels: - - conda-forge -dependencies: - - fenics=2019.1.0=py39hf3d152e_26 - - gmsh=4.6.0=hd134328_0 - - meshio=5.0.5=pyhd8ed1ab_0 - - paraview=5.9.1=hfc1cbd4_3_egl - - tectonic=0.8.0=ha1fef3e_1 - - cwltool diff --git a/exemplary_workflow/cwl/make_gmsh_mesh.cwl b/exemplary_workflow/cwl/make_gmsh_mesh.cwl deleted file mode 100644 index 730f29c5..00000000 --- a/exemplary_workflow/cwl/make_gmsh_mesh.cwl +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: CommandLineTool - -doc: Generate the computational mesh with gmsh - -hints: - SoftwareRequirement: - packages: - gmsh: - version: [ 4.6.0=hd134328_0, 4.6.0 ] - specs: - - https://anaconda.org/conda-forge/gmsh - - https://identifiers.org/rrid/RRID:SCR_021226 - -baseCommand: gmsh - -arguments: - - -setnumber - - domain_size - - $(inputs.domain_size) - - "-2" - - $(inputs.geofile.path) - - -o - - mesh.msh - -inputs: - geofile: - type: File - doc: "Geometry file in a gmsh-readable format" - default: - class: File - location: ../source/unit_square.geo - domain_size: - type: float - doc: "Specify the size of the domain to be meshed" - -outputs: - mesh: - type: File - outputBinding: - glob: mesh.msh diff --git a/exemplary_workflow/cwl/make_paraview_plot.cwl b/exemplary_workflow/cwl/make_paraview_plot.cwl deleted file mode 100644 index f06c15d6..00000000 --- a/exemplary_workflow/cwl/make_paraview_plot.cwl +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: CommandLineTool - -doc: Create plot-over-line data with paraview`s pvbatch - -baseCommand: [pvbatch] - -arguments: [ $(inputs.script), $(inputs.pvdfile.path), plotoverline.csv ] - -hints: - SoftwareRequirement: - packages: - paraview: - version: [ 5.9.1=hfc1cbd4_3_egl, 5.9.1 ] - specs: - - https://anaconda.org/conda-forge/paraview - - https://identifiers.org/rrid/RRID:SCR_002516 - - https://bio.tools/paraview - -requirements: - InitialWorkDirRequirement: - listing: - - $(inputs.vtkfile) - - $(inputs.pvdfile) - -inputs: - script: - type: File - default: - class: File - location: ../source/postprocessing.py - vtkfile: - type: File - pvdfile: - type: File - -outputs: - resultcsv: - type: File - format: https://www.iana.org/assignments/media-types/text/csv - outputBinding: - glob: plotoverline.csv diff --git a/exemplary_workflow/cwl/prepare_paper_macros.cwl b/exemplary_workflow/cwl/prepare_paper_macros.cwl deleted file mode 100644 index 9a3aaaa5..00000000 --- a/exemplary_workflow/cwl/prepare_paper_macros.cwl +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: CommandLineTool - -doc: Update the macros of the paper with values from this run - -baseCommand: python3 - -arguments: - - $(inputs.substitution_script) - - --domain-size - - $(inputs.domain_size) - - --macro-template-file - - $(inputs.macros_template) - - --output-macro-file - - macros.tex - - --num-dofs - - $(inputs.num_dofs) - - --plot-data-path - - $(inputs.plot_data_file.basename) - -requirements: - InitialWorkDirRequirement: - listing: - - $(inputs.substitution_script) - - $(inputs.macros_template) - -inputs: - domain_size: - type: float - num_dofs: - type: float - plot_data_file: - type: File - substitution_script: - type: File - default: - class: File - location: ../source/prepare_paper_macros.py - macros_template: - type: File - default: - class: File - location: ../source/macros.tex.template - -outputs: - macros_file: - type: File - outputBinding: - glob: macros.tex diff --git a/exemplary_workflow/cwl/run_dolfin.cwl b/exemplary_workflow/cwl/run_dolfin.cwl deleted file mode 100644 index 641527fb..00000000 --- a/exemplary_workflow/cwl/run_dolfin.cwl +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: CommandLineTool - -doc: Run the poisson solver in dolfin - -baseCommand: python3 - -arguments: - - $(inputs.script) - - --mesh - - $(inputs.xdmfmeshfile.path) - - --degree - - "2" - - --output - - result.pvd - -stdout: output.txt - -hints: - SoftwareRequirement: - packages: - fenics: - version: [ 2019.1.0=py39hf3d152e_26, 2019.1.0 ] - specs: - - https://anaconda.org/conda-forge/fenics - - https://bio.tools/fenics - -requirements: - InlineJavascriptRequirement: {} - InitialWorkDirRequirement: - listing: - - $(inputs.xdmfmeshfile) - - $(inputs.h5meshfile) - -inputs: - script: - type: File - default: - class: File - location: ../source/poisson.py - xdmfmeshfile: - type: File - h5meshfile: - type: File - -outputs: - resultvtu: - type: File - outputBinding: - glob: result000000.vtu - resultpvd: - type: File - outputBinding: - glob: result.pvd - num_dofs: - type: float - outputBinding: - glob: output.txt - loadContents: true - outputEval: | - ${ - var output = self[0].contents; - var dofs_string = output.split("Number of dofs used:")[1]; - dofs_string = dofs_string.split("\n")[0]; - return parseFloat(dofs_string); - } diff --git a/exemplary_workflow/cwl/wf_run_exemplary_wf.cwl b/exemplary_workflow/cwl/wf_run_exemplary_wf.cwl deleted file mode 100644 index 6f51b3b0..00000000 --- a/exemplary_workflow/cwl/wf_run_exemplary_wf.cwl +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 -class: Workflow - -outputs: - paperpdf: - type: File - format: iana:application/pdf - outputSource: compile_paper/pdf - pol_data: - type: File - format: iana:text/csv - outputSource: plot_over_line/resultcsv - macros: - type: File - outputSource: prepare_paper_macros/macros_file - resultvtu: - type: File - outputSource: run_simulation/resultvtu - -inputs: - domain_size: - type: float - default: 1.0 - -steps: - make_mesh: - run: make_gmsh_mesh.cwl - in: - domain_size: domain_size - out: [mesh] - - convert_mesh: - run: convert_msh_to_xdmf.cwl - in: - inputmesh: make_mesh/mesh - out: [outputmesh, outputmeshdata] - - run_simulation: - run: run_dolfin.cwl - in: - xdmfmeshfile: convert_mesh/outputmesh - h5meshfile: convert_mesh/outputmeshdata - out: [resultvtu, resultpvd, num_dofs] - - plot_over_line: - run: make_paraview_plot.cwl - in: - vtkfile: run_simulation/resultvtu - pvdfile: run_simulation/resultpvd - out: [resultcsv] - - prepare_paper_macros: - run: prepare_paper_macros.cwl - in: - num_dofs: run_simulation/num_dofs - domain_size: domain_size - plot_data_file: plot_over_line/resultcsv - out: [macros_file] - - compile_paper: - run: compile_paper.cwl - in: - csvfile: plot_over_line/resultcsv - macros: prepare_paper_macros/macros_file - out: [pdf] - -$namespaces: - iana: https://www.iana.org/assignments/media-types/ diff --git a/exemplary_workflow/cwl/workflow.svg b/exemplary_workflow/cwl/workflow.svg deleted file mode 100644 index 07e9a380..00000000 --- a/exemplary_workflow/cwl/workflow.svg +++ /dev/null @@ -1,189 +0,0 @@ - - - - - - -workflow - -cluster_outputs - -Workflow Outputs - - -cluster_inputs - -Workflow Inputs - - - -domain_size - -domain_size - - - -make_mesh - -make_mesh - - - -domain_size->make_mesh - - -domain_size - - - -prepare_paper_macros - -prepare_paper_macros - - - -domain_size->prepare_paper_macros - - -domain_size - - - -macros - -macros - - - -paperpdf - -paperpdf - - - -resultvtu - -resultvtu - - - -pol_data - -pol_data - - - -plot_over_line - -plot_over_line - - - -plot_over_line->pol_data - - - - - -compile_paper - -compile_paper - - - -plot_over_line->compile_paper - - -csvfile - - - -plot_over_line->prepare_paper_macros - - -plot_data_file - - - -convert_mesh - -convert_mesh - - - -make_mesh->convert_mesh - - -inputmesh - - - -compile_paper->paperpdf - - - - - -run_simulation - -run_simulation - - - -run_simulation->resultvtu - - - - - -run_simulation->plot_over_line - - -vtkfile - - - -run_simulation->plot_over_line - - -pvdfile - - - -run_simulation->prepare_paper_macros - - -num_dofs - - - -prepare_paper_macros->macros - - - - - -prepare_paper_macros->compile_paper - - -macros - - - -convert_mesh->run_simulation - - -h5meshfile - - - -convert_mesh->run_simulation - - -xdmfmeshfile - - - diff --git a/exemplary_workflow/dolfin.patch b/exemplary_workflow/dolfin.patch deleted file mode 100644 index 706a42f6..00000000 --- a/exemplary_workflow/dolfin.patch +++ /dev/null @@ -1,49 +0,0 @@ -diff --git a/dolfin/geometry/IntersectionConstruction.cpp b/dolfin/geometry/IntersectionConstruction.cpp -index 765dbb6..9c772e4 100644 ---- a/dolfin/geometry/IntersectionConstruction.cpp -+++ b/dolfin/geometry/IntersectionConstruction.cpp -@@ -19,6 +19,8 @@ - // Last changed: 2017-12-12 - - #include -+#include -+ - #include - #include "predicates.h" - #include "GeometryPredicates.h" -diff --git a/dolfin/io/VTKFile.cpp b/dolfin/io/VTKFile.cpp -index 2fee53b..7120687 100644 ---- a/dolfin/io/VTKFile.cpp -+++ b/dolfin/io/VTKFile.cpp -@@ -20,7 +20,6 @@ - #include - #include - #include --#include - - #include "pugixml.hpp" - -diff --git a/dolfin/io/VTKWriter.cpp b/dolfin/io/VTKWriter.cpp -index eff6934..b57a665 100644 ---- a/dolfin/io/VTKWriter.cpp -+++ b/dolfin/io/VTKWriter.cpp -@@ -24,7 +24,6 @@ - #include - #include - #include --#include - - #include - #include -diff --git a/dolfin/mesh/MeshFunction.h b/dolfin/mesh/MeshFunction.h -index 08cbc82..d314f85 100644 ---- a/dolfin/mesh/MeshFunction.h -+++ b/dolfin/mesh/MeshFunction.h -@@ -26,6 +26,7 @@ - - #include - #include -+#include - - #include - #include diff --git a/exemplary_workflow/gwl/README.md b/exemplary_workflow/gwl/README.md deleted file mode 100644 index d7947744..00000000 --- a/exemplary_workflow/gwl/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Workflow implementation with GWL - -[`GWL`](https://guixwl.org) requires a [`Guix`](https://guix.gnu.org) -installation on your system. - -The following steps assume that your current working directory is the -[gwl](.) directory of this repository: - -```sh -cd exemplary_workflow/gwl -``` - -## Using your Guix installation - -You are encouraged to execute the workflow with your current Guix installation -(potentially using recent packages). - -First, create a temporary environment with GWL: - -```sh -guix shell gwl -``` - -> If your Guix installation doesn't support the `shell` command, please upgrade -> your installation via `guix pull`. - -This will spawn a new shell, in which GWL is available. - -Next, enable the GWL extension. - -```sh -export GUIX_EXTENSIONS_PATH=$GUIX_ENVIRONMENT/share/guix/extensions -``` - -You should now be able to execute the workflow: - -``` -guix workflow run gwl/workflow.w -``` - -Subsequent runs should be much faster, since the computation environments for -each workflow step are cached. - -> Instead of working in a temporary environment, it may be convenient to install -> GWL into your default Guix profile using `guix package -i gwl` and fix the -> configuration in your shells startup files. - -## Using the authors Guix installation - -Instead of using your current and probably more recent Guix installation, you -can also jump back in time to use the known good Guix installation from the -author. - -> Please try this approach, if newer package versions in your Guix installation -> break this workflow. - -To do this, pull your Guix installation to the following commit hash: - -```sh -guix pull --commit=8e54584d4448d37ddf8ae995bb545a181ba2493c -``` - -Now you can follow the above commands. diff --git a/exemplary_workflow/gwl/source b/exemplary_workflow/gwl/source deleted file mode 120000 index e3787aab..00000000 --- a/exemplary_workflow/gwl/source +++ /dev/null @@ -1 +0,0 @@ -../source \ No newline at end of file diff --git a/exemplary_workflow/gwl/workflow.w b/exemplary_workflow/gwl/workflow.w deleted file mode 100644 index bb6837ec..00000000 --- a/exemplary_workflow/gwl/workflow.w +++ /dev/null @@ -1,111 +0,0 @@ -;; Variables -define domain-size 1 - - -;; Define workflow process templates -process make-gmsh-mesh (with geofile domain-size) - synopsis "Generate the computational mesh with gmsh" - packages "gmsh" - inputs geofile: geofile - outputs mesh: "mesh.msh" - # { gmsh -setnumber domainsize {{domain-size}} -2 {{inputs}} -o {{outputs}} } - - -process convert-msh-to-xdmf (with inputmesh) - synopsis "Convert the produced gmsh mesh to xdmf" - packages "python" "python-meshio" - inputs inputmesh: inputmesh - outputs - . xdmf: : file (basename inputmesh ".msh") "_converted.xdmf" - . h5: : file (basename inputmesh ".msh") "_converted.h5" - # { meshio convert {{inputs}} {{outputs:xdmf}} } - - -process run-dolfin (with xdmfmeshfile h5meshfile) - synopsis "Run the poisson solver in dolfin" - packages "python" "fenics" "pkg-config" "python-pkgconfig" "openmpi" - . "openssh" "gcc-toolchain" - inputs script: "source/poisson.py" xdmf: xdmfmeshfile h5: h5meshfile - outputs pvd: "result.pvd" vtu: "result000000.vtu" num-dof: "num-dof" - # { - python3 {{inputs:script}} --mesh {{inputs:xdmf}} --degree 2 \ - --output {{outputs:pvd}} --num-dofs {{outputs:num-dof}} - } - - -process make-paraview-plot (with pvd vtu) - synopsis "Create plot-over-line data with paraview's pvbatch" - packages "paraview" - inputs script: "source/postprocessing.py" pvd: pvd vtu: vtu - outputs csv: "plotoverline.csv" - # { pvbatch {{inputs:script}} {{inputs:pvd}} {{outputs:csv}} } - - -process prepare-paper-macros (with domain-size num_dofs plot_data_file) - synopsis "Update the macros of the paper with values from this run" - packages "python" - inputs substitution_script: "source/prepare_paper_macros.py" - . macros_template: "source/macros.tex.template" num_dofs: num_dofs - . plot_data_file: plot_data_file - outputs macros_file: "macros.tex" - # bash { - python3 {{inputs:substitution_script}} \ - --domain-size {{domain-size}} \ - --macro-template-file {{inputs:macros_template}} \ - --num-dofs $(<{{inputs:num_dofs}}) \ - --plot-data-path {{inputs:plot_data_file}} \ - --output-macro-file {{outputs:macros_file}} - } - -process compile-paper (with macros csvfile) - synopsis "Create the paper as pdf, given the produced csv file" - packages "tectonic" "coreutils" - inputs macros: macros csvfile: csvfile texfile: "source/paper.tex" - outputs "paper.pdf" - # { cp {{inputs:texfile}} paper.tex; tectonic paper.tex } - - -;; Define workflow process - -;; Workflow process templates are invoked with parameters. These are either -;; variables (see make-mesh) or some outputs picked from other workflow -;; processes. For example, convert-mesh picks the mesh output from the make-mesh -;; process. -define make-mesh - make-gmsh-mesh "source/unit_square.geo" domain-size - - -define convert-mesh - convert-msh-to-xdmf - pick mesh: : process-outputs make-mesh with-tags: - - -define run-simulation - run-dolfin - pick xdmf: : process-outputs convert-mesh with-tags: - pick h5: : process-outputs convert-mesh with-tags: - - -define plot-over-line - make-paraview-plot - pick pvd: : process-outputs run-simulation with-tags: - pick vtu: : process-outputs run-simulation with-tags: - - -define paper-macros - prepare-paper-macros domain-size - pick num-dof: : process-outputs run-simulation with-tags: - pick csv: : process-outputs plot-over-line with-tags: - - -define paper - compile-paper - pick macros_file: : process-outputs paper-macros with-tags: - pick csv: : process-outputs plot-over-line with-tags: - - -;; Define exemplary workflow by auto-connecting processes. -workflow exemplary-wf - processes - auto-connect make-mesh convert-mesh run-simulation plot-over-line - . paper-macros paper diff --git a/exemplary_workflow/kadistudio/README.md b/exemplary_workflow/kadistudio/README.md deleted file mode 100644 index 13a8b7db..00000000 --- a/exemplary_workflow/kadistudio/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# KadiStudio -Implementation of the exemplary workflow with KadiStudio. `KadiStudio` is a software component of the -Kadi4Mat research data infrastructure, that is dedicated to model, visualise and execute scientific workflows using a GUI. - -## Installation -`KadiStudio` can be installed using the debian package provided [here](https://bwsyncandshare.kit.edu/s/cJSZrE6fDTR6cLQ). -To execute a workflow in KadiStudio, the two software components `process-manager` and `process-engine` are -required. They are available as debian packages and can be installed using the following commands: - -```sh -wget https://gitlab.com/iam-cms/workflows/process-manager/-/jobs/artifacts/master/download?job=pack_deb -O process_manager.zip -wget https://gitlab.com/iam-cms/workflows/process-engine/-/jobs/1764662488/artifacts/download -O process_engine.zip -unzip process_*.zip -apt-get install --yes ./build/*.deb -``` - -The process-manager is a software component dedicated to orchestrate the execution of workflows. It does not read or interpret -workflow files but only delegates their execution to a suitable process-engine and checks the execution status frequently. This -serves as a layer of abstraction between the workflow editor KadiStudio and the process-engines and allows the use of different -process-engines depending on the current use case. - -The process-engine on the other hand is responsible for reading, interpreting and executing the workflow file that it is handed by -the process-manager. Different implementations of the process-engine are conceivable that are suitable for certain use cases such as -parallel execution, remote execution etc.. The currently available process-engine, used in this example, executes all tools contained -in the workflow file sequentially in the order derived from the green dependency ports visible in the image below. - -The tools used within KadiStudio are collected in the python library `workflow-nodes`, that can be installed with: - -```sh -pip install workflow-nodes -``` -This library is necessary, as KadiStudio needs a machine readable description of the used tools to be able to visualise them inside -the GUI. Adding this description to existing tools can be implemented using wrapper scripts. Already implemented wrapper scripts for -certain tools can be found inside the workflow-nodes library which can be extended by users as desired. A more detailed documentation -of the workflow-nodes project as well as the description of how to add new tools can be found [here](https://pypi.org/project/workflow-nodes/). - -## Creating workflows - -Workflows can be created in KadiStudio using a GUI to which tools can be added using a point-and-click interface. -In the gif below this is exemplified by the exemplary workflow. - -![workflow creation](workflowCreation.gif) - -Each added tool represents a command that is run when the workflow is executed. Using the input ports on the -left-hand side the corresponding tool can be parameterised. Connecting the green dependency ports of the tools -nodes defines the order of execution making the modelled workflow easier to grasp for the user. - -The complete workflow saved in the exemplary_workflow.flow file is shown in the following image. -![workflow creation](WorkflowNFDI4Ing.png) - -## Running the exemplary_workflow - -The execution of a workflow is recommended to be started inside the GUI of KadiStudio, as it offers the possiblity to easily -track its status and to interact with the workflow in case a user input is required. When starting a workflow inside KadiStudio the -following command is executed: - -```sh -process_manager start exemplary_workflow.flow -``` - -Thus, if KadiStudio is not available or its GUI cannot be displayed, it is also possible to start a workflow directly from a terminal -using the command above. Alternatively, it is also possible to start a workflow directly by passing it to a process-engine using -the command: - -```sh -process_engine run exemplary_workflow.flow -p /path/to/execution_folder -``` - -However, using the process-engine directly is not recommended, as the background tasks initiated by the process-manager when executing a -workflow are omitted in this case. This can lead to overwriting data when executing a workflow several times. To install the necessary -workflow nodes for the exemplary_workflow, please use the pip install command shown within the install_components.sh script, as some unreleased -nodes are utilised. - -### Command line execution -- Necessary configuration - -The workflow saved in the exemplary_workflow.flow file was specifically modelled to run inside the actions of this github project. -For this the variable GITHUB_WORKSPACE was used to reliably access the folder structure of the container that runs the workflow. -To be able to run the workflow manually it is therefore necessary to set this variable on your machine as well. This can -be done using the command: - -```sh -export GITHUB_WORKSPACE="/path/to/NFDI4IngScientificWorkflowRequirements" -``` - -This path might vary depending on your folder structure. Its only essential point is that it directs to the folder in which -the exemplary_workflow folder is located. When executing the exemplary_workflow workflow inside the GUI of KadiStudio, setting this variable -can be omitted as a check whether the $GITHUB_WORKSPACE is known is performed before execution. In case it is unknown, the user will -be prompted to enter the path to the NFDI4IngScientificWorkflowRequirements folder interactively. diff --git a/exemplary_workflow/kadistudio/WorkflowNFDI4Ing.png b/exemplary_workflow/kadistudio/WorkflowNFDI4Ing.png deleted file mode 100644 index 0e1df2cf..00000000 Binary files a/exemplary_workflow/kadistudio/WorkflowNFDI4Ing.png and /dev/null differ diff --git a/exemplary_workflow/kadistudio/exemplary_workflow.flow b/exemplary_workflow/kadistudio/exemplary_workflow.flow deleted file mode 100644 index 9cec14e2..00000000 --- a/exemplary_workflow/kadistudio/exemplary_workflow.flow +++ /dev/null @@ -1,1563 +0,0 @@ -{ - "connections": [ - { - "in_id": "{50178f55-52d9-4f93-8edb-665cb9b3ce29}", - "in_index": 0, - "out_id": "{a3626550-47b7-4e66-909d-be7af1089c03}", - "out_index": 0 - }, - { - "in_id": "{5b885a1a-c56d-48ef-9d88-346145394728}", - "in_index": 2, - "out_id": "{3e70152d-a817-48b4-9349-2b8f8922ceb5}", - "out_index": 0 - }, - { - "in_id": "{466b1417-c152-4b45-a6a6-b8af6514b108}", - "in_index": 1, - "out_id": "{4726b97d-1a06-4fad-bd3e-2f3f7d2263d5}", - "out_index": 0 - }, - { - "converter": { - "in": { - "id": "StringData", - "name": "String" - }, - "out": { - "id": "PipeData", - "name": "Pipe" - } - }, - "in_id": "{50178f55-52d9-4f93-8edb-665cb9b3ce29}", - "in_index": 1, - "out_id": "{a3626550-47b7-4e66-909d-be7af1089c03}", - "out_index": 1 - }, - { - "in_id": "{45cb400b-f610-476d-b798-60a5ec161521}", - "in_index": 4, - "out_id": "{7d5dd930-237c-409c-a5b1-8bad536f3451}", - "out_index": 0 - }, - { - "in_id": "{51251a77-d8f6-4777-b699-fdcddd41ff81}", - "in_index": 1, - "out_id": "{aef407ea-c4be-4e6c-bcad-e823fb8f0ae1}", - "out_index": 0 - }, - { - "in_id": "{5b885a1a-c56d-48ef-9d88-346145394728}", - "in_index": 1, - "out_id": "{fb63b7e6-5275-4432-aefa-1a718bc7792a}", - "out_index": 0 - }, - { - "in_id": "{dde60a88-8ecd-42aa-bc79-03471811dda1}", - "in_index": 4, - "out_id": "{ec224666-e7a0-44d9-b059-b62e445c9e0a}", - "out_index": 0 - }, - { - "in_id": "{51251a77-d8f6-4777-b699-fdcddd41ff81}", - "in_index": 0, - "out_id": "{dde60a88-8ecd-42aa-bc79-03471811dda1}", - "out_index": 0 - }, - { - "in_id": "{7c74bab5-3974-4462-baaa-a6f383a6e6f6}", - "in_index": 1, - "out_id": "{4726b97d-1a06-4fad-bd3e-2f3f7d2263d5}", - "out_index": 0 - }, - { - "converter": { - "in": { - "id": "StringData", - "name": "String" - }, - "out": { - "id": "PipeData", - "name": "Pipe" - } - }, - "in_id": "{c540a6ed-4773-40d8-b899-2d50b6a71a8a}", - "in_index": 2, - "out_id": "{ac88f89d-1f94-4a0f-9f69-3d22106e4563}", - "out_index": 1 - }, - { - "in_id": "{45cb400b-f610-476d-b798-60a5ec161521}", - "in_index": 2, - "out_id": "{1c0f1152-497a-4c02-8a35-148030fa6576}", - "out_index": 0 - }, - { - "in_id": "{dde60a88-8ecd-42aa-bc79-03471811dda1}", - "in_index": 6, - "out_id": "{c6e92adc-6b66-460f-9d10-51462861632f}", - "out_index": 0 - }, - { - "in_id": "{4a2533f8-f89f-481f-bb0f-666f321e4c83}", - "in_index": 3, - "out_id": "{f4c3e069-b74e-489b-a0eb-cde5969b780e}", - "out_index": 0 - }, - { - "in_id": "{7c74bab5-3974-4462-baaa-a6f383a6e6f6}", - "in_index": 2, - "out_id": "{71a45946-a6d9-45a9-b9eb-0007e3202cfe}", - "out_index": 1 - }, - { - "in_id": "{71a45946-a6d9-45a9-b9eb-0007e3202cfe}", - "in_index": 1, - "out_id": "{96d05c5d-1558-402d-9cc9-91bc117d6a23}", - "out_index": 0 - }, - { - "in_id": "{4a2533f8-f89f-481f-bb0f-666f321e4c83}", - "in_index": 0, - "out_id": "{47b35abc-d775-4bb9-8cc7-cc6465e60746}", - "out_index": 0 - }, - { - "in_id": "{51251a77-d8f6-4777-b699-fdcddd41ff81}", - "in_index": 2, - "out_id": "{08ae43e4-fdfb-4828-bd59-55b542314f59}", - "out_index": 0 - }, - { - "in_id": "{4a2533f8-f89f-481f-bb0f-666f321e4c83}", - "in_index": 2, - "out_id": "{36ff6142-7f0c-413f-93d3-46d677fda3b6}", - "out_index": 0 - }, - { - "in_id": "{a3626550-47b7-4e66-909d-be7af1089c03}", - "in_index": 1, - "out_id": "{0bbe6c8f-8324-474b-aa96-d00f5122e5d2}", - "out_index": 0 - }, - { - "in_id": "{dde60a88-8ecd-42aa-bc79-03471811dda1}", - "in_index": 0, - "out_id": "{3d10a3df-0f4e-49f7-8bca-3306c893645b}", - "out_index": 0 - }, - { - "in_id": "{dde60a88-8ecd-42aa-bc79-03471811dda1}", - "in_index": 5, - "out_id": "{aef407ea-c4be-4e6c-bcad-e823fb8f0ae1}", - "out_index": 0 - }, - { - "in_id": "{45cb400b-f610-476d-b798-60a5ec161521}", - "in_index": 0, - "out_id": "{4a2533f8-f89f-481f-bb0f-666f321e4c83}", - "out_index": 0 - }, - { - "converter": { - "in": { - "id": "BooleanData", - "name": "bool" - }, - "out": { - "id": "PipeData", - "name": "Pipe" - } - }, - "in_id": "{3d10a3df-0f4e-49f7-8bca-3306c893645b}", - "in_index": 1, - "out_id": "{50178f55-52d9-4f93-8edb-665cb9b3ce29}", - "out_index": 1 - }, - { - "in_id": "{47b35abc-d775-4bb9-8cc7-cc6465e60746}", - "in_index": 0, - "out_id": "{51251a77-d8f6-4777-b699-fdcddd41ff81}", - "out_index": 0 - }, - { - "in_id": "{45cb400b-f610-476d-b798-60a5ec161521}", - "in_index": 0, - "out_id": "{c540a6ed-4773-40d8-b899-2d50b6a71a8a}", - "out_index": 0 - }, - { - "in_id": "{dde60a88-8ecd-42aa-bc79-03471811dda1}", - "in_index": 2, - "out_id": "{6ecad8d9-9983-437d-957f-97d5ccf90a59}", - "out_index": 0 - }, - { - "in_id": "{3d10a3df-0f4e-49f7-8bca-3306c893645b}", - "in_index": 0, - "out_id": "{50178f55-52d9-4f93-8edb-665cb9b3ce29}", - "out_index": 0 - }, - { - "in_id": "{ac88f89d-1f94-4a0f-9f69-3d22106e4563}", - "in_index": 1, - "out_id": "{220f41ef-264b-43e5-b5e7-15d954f30563}", - "out_index": 0 - }, - { - "in_id": "{8cdb1559-fb47-4df9-a8c8-9e26791c0a78}", - "in_index": 1, - "out_id": "{8eaca792-cd22-4f78-9cc9-ee97828f7299}", - "out_index": 0 - }, - { - "in_id": "{71a45946-a6d9-45a9-b9eb-0007e3202cfe}", - "in_index": 0, - "out_id": "{3d10a3df-0f4e-49f7-8bca-3306c893645b}", - "out_index": 2 - }, - { - "in_id": "{4a2533f8-f89f-481f-bb0f-666f321e4c83}", - "in_index": 1, - "out_id": "{44928488-49e4-4f55-816e-308501509049}", - "out_index": 0 - }, - { - "in_id": "{47b35abc-d775-4bb9-8cc7-cc6465e60746}", - "in_index": 2, - "out_id": "{4036635d-a7a4-43ba-a042-692e135df611}", - "out_index": 0 - }, - { - "in_id": "{c540a6ed-4773-40d8-b899-2d50b6a71a8a}", - "in_index": 0, - "out_id": "{ac88f89d-1f94-4a0f-9f69-3d22106e4563}", - "out_index": 0 - }, - { - "in_id": "{47b35abc-d775-4bb9-8cc7-cc6465e60746}", - "in_index": 4, - "out_id": "{bc0efa57-8e71-4711-b9b2-e276b38f7ddf}", - "out_index": 0 - }, - { - "in_id": "{466b1417-c152-4b45-a6a6-b8af6514b108}", - "in_index": 0, - "out_id": "{3d10a3df-0f4e-49f7-8bca-3306c893645b}", - "out_index": 1 - }, - { - "in_id": "{ac88f89d-1f94-4a0f-9f69-3d22106e4563}", - "in_index": 0, - "out_id": "{47b35abc-d775-4bb9-8cc7-cc6465e60746}", - "out_index": 0 - }, - { - "in_id": "{c540a6ed-4773-40d8-b899-2d50b6a71a8a}", - "in_index": 1, - "out_id": "{3f575615-5791-43ac-bdd6-0f9420f99ebc}", - "out_index": 0 - }, - { - "in_id": "{466b1417-c152-4b45-a6a6-b8af6514b108}", - "in_index": 2, - "out_id": "{0e9a7335-81c4-4008-91c6-9e8d1f39c113}", - "out_index": 0 - }, - { - "in_id": "{45cb400b-f610-476d-b798-60a5ec161521}", - "in_index": 1, - "out_id": "{4c007f80-696e-4dfd-a4ab-7c33a40e8136}", - "out_index": 0 - }, - { - "in_id": "{7c74bab5-3974-4462-baaa-a6f383a6e6f6}", - "in_index": 0, - "out_id": "{71a45946-a6d9-45a9-b9eb-0007e3202cfe}", - "out_index": 0 - }, - { - "in_id": "{8cdb1559-fb47-4df9-a8c8-9e26791c0a78}", - "in_index": 0, - "out_id": "{45cb400b-f610-476d-b798-60a5ec161521}", - "out_index": 0 - }, - { - "in_id": "{47b35abc-d775-4bb9-8cc7-cc6465e60746}", - "in_index": 1, - "out_id": "{b2080c1d-6d75-4fa6-a97e-52939d623514}", - "out_index": 0 - }, - { - "in_id": "{5b885a1a-c56d-48ef-9d88-346145394728}", - "in_index": 0, - "out_id": "{8cdb1559-fb47-4df9-a8c8-9e26791c0a78}", - "out_index": 0 - } - ], - "nodes": [ - { - "id": "{6ecad8d9-9983-437d-957f-97d5ccf90a59}", - "model": { - "name": "Boolean", - "value": "1" - }, - "position": { - "x": 310, - "y": 365 - } - }, - { - "id": "{f4c3e069-b74e-489b-a0eb-cde5969b780e}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/plotoverline.csv" - }, - "position": { - "x": 1772.7247058823523, - "y": 191.09647058823523 - } - }, - { - "id": "{7d5dd930-237c-409c-a5b1-8bad536f3451}", - "model": { - "name": "String", - "value": "python" - }, - "position": { - "x": 2290.7265429783824, - "y": 109.3301304603319 - } - }, - { - "id": "{ac88f89d-1f94-4a0f-9f69-3d22106e4563}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes system cat", - "path": "workflow-nodes system cat ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "arg0", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 2, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 3, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.14.0" - } - }, - "position": { - "x": 2174.8723072320577, - "y": 662.461105076431 - } - }, - { - "id": "{0bbe6c8f-8324-474b-aa96-d00f5122e5d2}", - "model": { - "name": "String", - "value": "$GITHUB_WORKSPACE" - }, - "position": { - "x": -1922.9897330350784, - "y": 333.5451218361729 - } - }, - { - "id": "{8cdb1559-fb47-4df9-a8c8-9e26791c0a78}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes misc tectonic", - "path": "workflow-nodes misc tectonic ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "arg0", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 2, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 3, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.11.0" - } - }, - "position": { - "x": 3162.354934177241, - "y": 278.19138465576265 - } - }, - { - "id": "{4c007f80-696e-4dfd-a4ab-7c33a40e8136}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/prepare_paper_macros.py" - }, - "position": { - "x": 2290.6502591114613, - "y": -32.92533086161872 - } - }, - { - "id": "{44928488-49e4-4f55-816e-308501509049}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/postprocessing.py" - }, - "position": { - "x": 1772.7247058823527, - "y": 52.85647058823527 - } - }, - { - "id": "{3e70152d-a817-48b4-9349-2b8f8922ceb5}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/kadistudio" - }, - "position": { - "x": 3211.1821175206046, - "y": 187.42308078914044 - } - }, - { - "id": "{1c0f1152-497a-4c02-8a35-148030fa6576}", - "model": { - "name": "String", - "value": "--macro-template-file ${base}/exemplary_workflow/source/macros.tex.template --plot-data-path ${base}/exemplary_workflow/source/plotoverline.csv --domain-size 4 --num-dofs ${numdofs} --output-macro-file ${base}/exemplary_workflow/source/macros.tex" - }, - "position": { - "x": 2290.4287348872485, - "y": 35.8976589766398 - } - }, - { - "id": "{45cb400b-f610-476d-b798-60a5ec161521}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes misc run-script", - "path": "workflow-nodes misc run-script ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "arg0", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "arguments", - "port_direction": "in", - "port_index": 2, - "position": 1, - "required": false, - "shortName": "a", - "type": "string" - }, - { - "name": "execute-in", - "port_direction": "in", - "port_index": 3, - "position": 2, - "required": false, - "shortName": "e", - "type": "string" - }, - { - "name": "interpreter", - "port_direction": "in", - "port_index": 4, - "position": 3, - "required": false, - "shortName": "i", - "type": "string" - }, - { - "name": "ignore-exitcode", - "port_direction": "in", - "port_index": 5, - "position": 4, - "required": false, - "shortName": "", - "type": "flag" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 6, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 7, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.11.0" - } - }, - "position": { - "x": 2721.209035154627, - "y": 277.73633766497306 - } - }, - { - "id": "{0e9a7335-81c4-4008-91c6-9e8d1f39c113}", - "model": { - "name": "String", - "value": "$GITHUB_WORKSPACE" - }, - "position": { - "x": -820.4378025210625, - "y": 546.407038450159 - } - }, - { - "id": "{a3626550-47b7-4e66-909d-be7af1089c03}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes system echo", - "path": "workflow-nodes system echo ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "arg0", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "no-newline", - "port_direction": "in", - "port_index": 2, - "position": 1, - "required": false, - "shortName": "n", - "type": "flag" - }, - { - "name": "enable-backslash-escapes", - "port_direction": "in", - "port_index": 3, - "position": 2, - "required": false, - "shortName": "e", - "type": "flag" - }, - { - "name": "disable-backslash-escapes", - "port_direction": "in", - "port_index": 4, - "position": 3, - "required": false, - "shortName": "E", - "type": "flag" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 5, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 6, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.14.0" - } - }, - "position": { - "x": -1529.0321018867417, - "y": 280.506909435353 - } - }, - { - "id": "{dde60a88-8ecd-42aa-bc79-03471811dda1}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes misc gmsh", - "path": "workflow-nodes misc gmsh ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "one-dimensional-mesh", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "a", - "type": "flag" - }, - { - "name": "two-dimensional-mesh", - "port_direction": "in", - "port_index": 2, - "position": 1, - "required": false, - "shortName": "b", - "type": "flag" - }, - { - "name": "three-dimensional-mesh", - "port_direction": "in", - "port_index": 3, - "position": 2, - "required": false, - "shortName": "c", - "type": "flag" - }, - { - "name": "geo-file", - "port_direction": "in", - "port_index": 4, - "position": 3, - "required": true, - "shortName": "g", - "type": "string" - }, - { - "name": "file", - "port_direction": "in", - "port_index": 5, - "position": 4, - "required": true, - "shortName": "o", - "type": "string" - }, - { - "name": "set-number", - "port_direction": "in", - "port_index": 6, - "position": 5, - "required": false, - "shortName": "n", - "type": "string" - }, - { - "name": "save-and-exit", - "port_direction": "in", - "port_index": 7, - "position": 6, - "required": false, - "shortName": "s", - "type": "flag" - }, - { - "name": "format", - "port_direction": "in", - "port_index": 8, - "position": 7, - "required": false, - "shortName": "f", - "type": "string" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 9, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 10, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.11.0" - } - }, - "position": { - "x": 447, - "y": 280 - } - }, - { - "id": "{3f575615-5791-43ac-bdd6-0f9420f99ebc}", - "model": { - "name": "String", - "value": "numdofs" - }, - "position": { - "x": 2237.0675503725147, - "y": 566.0964696668576 - } - }, - { - "id": "{466b1417-c152-4b45-a6a6-b8af6514b108}", - "model": { - "executionProfile": "Default", - "name": "Variable" - }, - "position": { - "x": -412.9606270352618, - "y": 416.64777549093355 - } - }, - { - "id": "{aef407ea-c4be-4e6c-bcad-e823fb8f0ae1}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/square_mesh.msh" - }, - "position": { - "x": 100, - "y": 486 - } - }, - { - "id": "{ec224666-e7a0-44d9-b059-b62e445c9e0a}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/unit_square.geo" - }, - "position": { - "x": 99.99999999999999, - "y": 427 - } - }, - { - "id": "{7c74bab5-3974-4462-baaa-a6f383a6e6f6}", - "model": { - "executionProfile": "Default", - "name": "Variable" - }, - "position": { - "x": -141.24614853266507, - "y": 604.1250931216398 - } - }, - { - "id": "{8eaca792-cd22-4f78-9cc9-ee97828f7299}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/paper.tex" - }, - "position": { - "x": 2818.012098600341, - "y": 181.34918465245534 - } - }, - { - "id": "{96d05c5d-1558-402d-9cc9-91bc117d6a23}", - "model": { - "name": "String", - "value": "Please select the driectory in which the exemplary_workflow folder is located." - }, - "position": { - "x": -825.3540004619584, - "y": 660.2832003695669 - } - }, - { - "id": "{4036635d-a7a4-43ba-a042-692e135df611}", - "model": { - "name": "String", - "value": "-m ${base}/exemplary_workflow/source/square.xdmf -d 2 -o ${base}/exemplary_workflow/source/poisson.pvd -n ${base}/exemplary_workflow/source/numdofs.txt" - }, - "position": { - "x": 1117.176470588235, - "y": 91.2941176470588 - } - }, - { - "id": "{47b35abc-d775-4bb9-8cc7-cc6465e60746}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes misc run-script", - "path": "workflow-nodes misc run-script ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "arg0", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "arguments", - "port_direction": "in", - "port_index": 2, - "position": 1, - "required": false, - "shortName": "a", - "type": "string" - }, - { - "name": "execute-in", - "port_direction": "in", - "port_index": 3, - "position": 2, - "required": false, - "shortName": "e", - "type": "string" - }, - { - "name": "interpreter", - "port_direction": "in", - "port_index": 4, - "position": 3, - "required": false, - "shortName": "i", - "type": "string" - }, - { - "name": "ignore-exitcode", - "port_direction": "in", - "port_index": 5, - "position": 4, - "required": false, - "shortName": "", - "type": "flag" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 6, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 7, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.11.0" - } - }, - "position": { - "x": 1657.2536470588234, - "y": 278.1063529411764 - } - }, - { - "id": "{3d10a3df-0f4e-49f7-8bca-3306c893645b}", - "model": { - "executionProfile": "Default", - "name": "IfBranch" - }, - "position": { - "x": -715.0231735960814, - "y": 281.92504345676525 - } - }, - { - "id": "{08ae43e4-fdfb-4828-bd59-55b542314f59}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/square.xdmf" - }, - "position": { - "x": 573.7411764705881, - "y": 170.54117647058823 - } - }, - { - "id": "{b2080c1d-6d75-4fa6-a97e-52939d623514}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/poisson.py" - }, - "position": { - "x": 1115.2941176470588, - "y": 19.764705882352956 - } - }, - { - "id": "{71a45946-a6d9-45a9-b9eb-0007e3202cfe}", - "model": { - "executionProfile": "Default", - "name": "UserInputFile" - }, - "position": { - "x": -416.93140229521623, - "y": 604.1250931216398 - } - }, - { - "id": "{4a2533f8-f89f-481f-bb0f-666f321e4c83}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes misc pvbatch", - "path": "workflow-nodes misc pvbatch ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "script", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "s", - "type": "string" - }, - { - "name": "vtk-file", - "port_direction": "in", - "port_index": 2, - "position": 1, - "required": true, - "shortName": "v", - "type": "string" - }, - { - "name": "output-file", - "port_direction": "in", - "port_index": 3, - "position": 2, - "required": false, - "shortName": "o", - "type": "string" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 4, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 5, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.11.0" - } - }, - "position": { - "x": 2162.884845361455, - "y": 277.4248375223784 - } - }, - { - "id": "{220f41ef-264b-43e5-b5e7-15d954f30563}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/numdofs.txt" - }, - "position": { - "x": 1743.4068086012194, - "y": 693.0255172445294 - } - }, - { - "id": "{36ff6142-7f0c-413f-93d3-46d677fda3b6}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/poisson.pvd" - }, - "position": { - "x": 1772.7247058823523, - "y": 123.33176470588234 - } - }, - { - "id": "{5b885a1a-c56d-48ef-9d88-346145394728}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes system cp", - "path": "workflow-nodes system cp ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "arg0", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "arg1", - "port_direction": "in", - "port_index": 2, - "position": 1, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "repository", - "port_direction": "in", - "port_index": 3, - "position": 2, - "required": false, - "shortName": "", - "type": "flag" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 4, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 5, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.11.0" - } - }, - "position": { - "x": 3608.76200463905, - "y": 277.663823391319 - } - }, - { - "id": "{50178f55-52d9-4f93-8edb-665cb9b3ce29}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes misc boolconvert", - "path": "workflow-nodes misc boolconvert ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "expression", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "e", - "type": "string" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 2, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 3, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.14.0" - } - }, - "position": { - "x": -1113.5188336129377, - "y": 280.5069094353528 - } - }, - { - "id": "{4726b97d-1a06-4fad-bd3e-2f3f7d2263d5}", - "model": { - "name": "String", - "value": "base" - }, - "position": { - "x": -821.3832252020039, - "y": 485.190919859195 - } - }, - { - "id": "{51251a77-d8f6-4777-b699-fdcddd41ff81}", - "model": { - "executionProfile": "Default", - "name": "ToolNode", - "tool": { - "name": "workflow-nodes misc meshio-convert", - "path": "workflow-nodes misc meshio-convert ", - "ports": [ - { - "name": "Dependencies", - "port_direction": "in", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "arg0", - "port_direction": "in", - "port_index": 1, - "position": 0, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "arg1", - "port_direction": "in", - "port_index": 2, - "position": 1, - "required": true, - "shortName": "", - "type": "string" - }, - { - "name": "input-format", - "port_direction": "in", - "port_index": 3, - "position": 2, - "required": false, - "shortName": "f", - "type": "string" - }, - { - "name": "output-format", - "port_direction": "in", - "port_index": 4, - "position": 3, - "required": false, - "shortName": "v", - "type": "string" - }, - { - "name": "prune", - "port_direction": "in", - "port_index": 5, - "position": 4, - "required": false, - "shortName": "p", - "type": "flag" - }, - { - "name": "environment", - "port_direction": "in", - "port_index": 6, - "position": 0, - "required": false, - "shortName": "", - "type": "env" - }, - { - "name": "stdin", - "port_direction": "in", - "port_index": 7, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - }, - { - "name": "Dependents", - "port_direction": "out", - "port_index": 0, - "position": 0, - "required": false, - "shortName": "", - "type": "dependency" - }, - { - "name": "stdout", - "port_direction": "out", - "port_index": 1, - "position": 0, - "required": false, - "shortName": "", - "type": "pipe" - } - ], - "version": "0.11.0" - } - }, - "position": { - "x": 998.4000000000001, - "y": 278.9647058823528 - } - }, - { - "id": "{c540a6ed-4773-40d8-b899-2d50b6a71a8a}", - "model": { - "executionProfile": "Default", - "name": "Variable" - }, - "position": { - "x": 2609.203889145474, - "y": 662.9247666232799 - } - }, - { - "id": "{c6e92adc-6b66-460f-9d10-51462861632f}", - "model": { - "name": "String", - "value": "domain_size 4" - }, - "position": { - "x": 100, - "y": 549 - } - }, - { - "id": "{fb63b7e6-5275-4432-aefa-1a718bc7792a}", - "model": { - "name": "String", - "value": "${base}/exemplary_workflow/source/paper.pdf" - }, - "position": { - "x": 3209.0996388451704, - "y": 115.40402659701695 - } - }, - { - "id": "{bc0efa57-8e71-4711-b9b2-e276b38f7ddf}", - "model": { - "name": "String", - "value": "python" - }, - "position": { - "x": 1116.0784313725487, - "y": 166.2745098039215 - } - } - ] -} - diff --git a/exemplary_workflow/kadistudio/install_components.sh b/exemplary_workflow/kadistudio/install_components.sh deleted file mode 100644 index 06465813..00000000 --- a/exemplary_workflow/kadistudio/install_components.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -apt-get install -y git -python3 -m pip install git+https://gitlab.com/lars153/workflow-nodes.git@b1447ca3ad8b8b79d2153ca142d3141f2a0c63f2 -wget https://gitlab.com/iam-cms/workflows/process-engine/-/jobs/1764662488/artifacts/download -O process_engine.zip -unzip process_engine.zip -sudo apt-get install --yes ./build/*.deb -rm build/*.deb diff --git a/exemplary_workflow/kadistudio/workflowCreation.gif b/exemplary_workflow/kadistudio/workflowCreation.gif deleted file mode 100644 index cdd8abf1..00000000 Binary files a/exemplary_workflow/kadistudio/workflowCreation.gif and /dev/null differ diff --git a/exemplary_workflow/nextflow/.gitignore b/exemplary_workflow/nextflow/.gitignore deleted file mode 100644 index d26af960..00000000 --- a/exemplary_workflow/nextflow/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -work -paper.pdf -.nextflow -.nextflow.log* -tutorial.nf diff --git a/exemplary_workflow/nextflow/README.md b/exemplary_workflow/nextflow/README.md deleted file mode 100644 index 6e7b59e0..00000000 --- a/exemplary_workflow/nextflow/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Nextflow -This directory contains an implementation of the exemplary workflow with [nextflow](https://nextflow.io/). -The documentation is available under [this link](https://www.nextflow.io/docs/latest/index.html). - -## Installation -Nextflow is distributed as a self-installing package, which means that it does not require any special installation procedure. -You can just download the executable package following the instructions in the [documentation](https://www.nextflow.io/docs/latest/getstarted.html#installation). -Here we create a dedicated conda environment to run the exemplary workflow with nextflow. -``` -conda create --name nextflow --channel bioconda nextflow=21.04.0 -``` - -## Running the exemplary workflow -Executing the workflow is as simple as -``` -nextflow run exemplarywf.nf -``` -Nextflow will install all required software specified through conda environment files for you. -Any script parameter can be specified on the command line by prefixing the parameter name with -double dash characters, e.g. -``` -nextflow run exemplarywf.nf --domainSize 1.0 -``` diff --git a/exemplary_workflow/nextflow/exemplarywf.nf b/exemplary_workflow/nextflow/exemplarywf.nf deleted file mode 100644 index 5ace1c6b..00000000 --- a/exemplary_workflow/nextflow/exemplarywf.nf +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env nextflow - -params.domainSize = 1.0 - -// for now simply add dependent files as Channels -geofile = Channel.fromPath("$PWD/../source/unit_square.geo") -fenics_code = Channel.fromPath("$PWD/../source/poisson.py") -paraview_script = Channel.fromPath("$PWD/../source/postprocessing.py") -macro_template = Channel.fromPath("$PWD/../source/macros.tex.template") -prepare_macros = Channel.fromPath("$PWD/../source/prepare_paper_macros.py") -paper_source = Channel.fromPath("$PWD/../source/paper.tex") - - -process generateMesh { - // generate mesh using Gmsh - - // use conda directive to specify environment file - conda "../source/envs/preprocessing.yaml" - - input: - file geofile - - output: - file "unit_square.msh" into mesh_msh - - """ - gmsh -2 -setnumber domain_size ${params.domainSize} $geofile -o unit_square.msh - """ -} - - -process convertToXDMF { - // convert .msh file to .xdmf - - conda "../source/envs/preprocessing.yaml" - - input: - file mesh from mesh_msh - - output: - file "unit_square.h5" into mesh_h5 - file "unit_square.xdmf" into mesh_xdmf - - """ - meshio convert $mesh unit_square.xdmf - """ -} - - -process solvePoisson { - // solve poisson equation using fenics - - conda "../source/envs/processing.yaml" - - // multiple inputs from different channels - input: - file fenics_code - file meshdata from mesh_h5 - file mesh from mesh_xdmf - - output: - file "poisson.pvd" into poisson_pvd - file "poisson*.vtu" into poisson_vtu - stdout poisson_stdout - - """ - python $fenics_code --mesh $mesh --degree 2 --outputfile poisson.pvd - """ -} - - -process readNumberOfDofs { - input: - val x from poisson_stdout - - output: - stdout number_of_dofs - - """ - #!/usr/bin/python3 - s = '''${x.replaceAll("\\n", "&")}''' - dof_string = s.split("Number of dofs used:")[1] - num_dofs = dof_string.split("&")[0] - print(int(num_dofs)) - """ -} - - -process makePlotOverLine { - - conda "../source/envs/postprocessing.yaml" - - input: - file paraview_script - file vtu from poisson_vtu - file pvd from poisson_pvd - - output: - file "plotoverline.csv" into (plot_over_line, plotOverLine) - - """ - pvbatch $paraview_script $pvd plotoverline.csv - """ -} - - -process substituteMacros { - // place the correct value into the paper macros - - conda "../source/envs/postprocessing.yaml" - - input: - file prepare_macros - file macro_template - file plotOverLine - val number_of_dofs - - output: - file "macros.tex" into macros - - """ - python $prepare_macros --macro-template-file $macro_template \ - --plot-data-path $plotOverLine \ - --domain-size ${params.domainSize} \ - --num-dofs ${number_of_dofs.replaceAll("\\s", "")} \ - --output-macro-file macros.tex - """ -} - - -process compilePaper { - - conda "../source/envs/postprocessing.yaml" - publishDir "$PWD" - - input: - file paper_source - file plot_over_line - file macros - - output: - file "paper.pdf" - - """ - tectonic $paper_source - """ -} diff --git a/exemplary_workflow/pydoit/.gitignore b/exemplary_workflow/pydoit/.gitignore deleted file mode 100644 index 097ef694..00000000 --- a/exemplary_workflow/pydoit/.gitignore +++ /dev/null @@ -1,18 +0,0 @@ -# SimpleUseCase: generated files -*.xdmf -*.h5 -*.msh -*.csv -*.png -*.pvd -*.vtu -*.txt - -# latex -*.tex -*.aux -*.fdb_latexmk -*.fls -*.log -*.synctex.gz -*.pdf diff --git a/exemplary_workflow/pydoit/README.md b/exemplary_workflow/pydoit/README.md deleted file mode 100644 index 5ce2d474..00000000 --- a/exemplary_workflow/pydoit/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Pydoit -This directory contains an implementation of the exemplary workflow with [pydoit](https://pydoit.org). - -## Installation -Doit can be installed with `pip` or `conda`. -``` -pip install doit=0.33.1 -``` -``` -conda install --channel conda-forge doit=0.33.1 -``` -However, it does not handle the installation of the required software for you. -Here, we assume that all required software and `doit` are already installed. -For example you could do -``` -conda env create --name doit_env --file ../source/envs/default_env.yaml -``` -to create a conda environment with all required software and then install `doit` in the -same environment with -``` -conda install --name doit_env --channel conda-forge doit=0.33.1 -``` - - -## Running the exemplary workflow -If you are using `conda`, activate your environment. -``` -conda activate doit_env -``` -Then run the workflow with -``` -doit -``` diff --git a/exemplary_workflow/pydoit/dodo.py b/exemplary_workflow/pydoit/dodo.py deleted file mode 100644 index 9a0b07bf..00000000 --- a/exemplary_workflow/pydoit/dodo.py +++ /dev/null @@ -1,151 +0,0 @@ -import pathlib -from doit import get_var -from doit.action import CmdAction -from doit.tools import config_changed - -ROOT = pathlib.Path(__file__).parent -SOURCE = ROOT.parent / "source" - -GLOBAL_PARAMS = {"size": get_var("size", "2.0")} -DOMAIN_SIZE = GLOBAL_PARAMS["size"] - -DOIT_CONFIG = { - "action_string_formatting": "both", - "verbosity": 2, -} - - -def task_generate_mesh(): - """generate the mesh with Gmsh""" - geo = SOURCE / "unit_square.geo" - msh = ROOT / "unit_square.msh" - args = [ - "gmsh", - "-2", - "-setnumber", - "domain_size", - f"{DOMAIN_SIZE}", - f"{geo}", - "-o", - f"{msh}", - ] - return { - "file_dep": [geo], - "actions": [CmdAction(" ".join(args), save_out="stdout")], - "targets": [msh], - "clean": True, - "uptodate": [config_changed(DOMAIN_SIZE)], - } - - -def task_get_domain_size(): - """parse stdout of the task `task_generate_mesh` and return domain size as float""" - - def parse(stdout): - s = stdout.split("Used domain size:")[1] - size = float(s.split("Used mesh size")[0]) - return {"size": size} - - return { - "actions": [(parse,)], - "getargs": {"stdout": ("generate_mesh", "stdout")}, - } - - -def task_convert(): - """convert the mesh file from msh to xdmf format""" - msh = ROOT / "unit_square.msh" - return { - "file_dep": [msh], - "actions": [f"meshio convert {msh} {msh.with_suffix('.xdmf')}"], - "targets": [msh.with_suffix(".xdmf"), msh.with_suffix(".h5")], - "clean": True, - } - - -def task_poisson(): - """solve the poisson equation with fenics""" - poisson = SOURCE / "poisson.py" - mesh = ROOT / "unit_square.xdmf" - degree = 2 - pvdfile = ROOT / "poisson.pvd" - vtufile = ROOT / (pvdfile.stem + "000000.vtu") - txtfile = ROOT / "numdofs.txt" - return { - "file_dep": [mesh, mesh.with_suffix(".h5"), poisson], - "actions": [ - f"python {poisson} --mesh {mesh} --degree {degree} --outputfile {pvdfile} --num-dofs {txtfile}" - ], - "targets": [pvdfile, vtufile, txtfile], - "clean": True, - } - - -def task_get_num_dofs(): - """reads number of degrees of freedom and returns it as integer""" - - def read(dependencies): - with open(dependencies[0], "r") as handle: - num_dofs = int(handle.read()) - - return {"num_dofs": num_dofs} - - return { - "file_dep": [ROOT / "numdofs.txt"], - "actions": [(read,)], - } - - -def task_plot_over_line(): - """write data using Paraview""" - postproc = SOURCE / "postprocessing.py" - pvdfile = ROOT / "poisson.pvd" - vtufile = ROOT / (pvdfile.stem + "000000.vtu") - pol = ROOT / "plotoverline.csv" - return { - "file_dep": [pvdfile, vtufile, postproc], - "actions": [f"pvbatch {postproc} {pvdfile} {pol}"], - "targets": [pol], - "clean": True, - } - - -def task_substitute_macros(): - """places the correct values into the paper macros""" - script = SOURCE / "prepare_paper_macros.py" - file_deps = { - "--macro-template-file": SOURCE / "macros.tex.template", - "--plot-data-path": ROOT / "plotoverline.csv", - } - - def create_cmd(): - cmd = f"python {script}" - for key, value in file_deps.items(): - cmd += f" {key} {value}" - cmd += " --domain-size {size}" - cmd += " --num-dofs {num_dofs}" - cmd += " --output-macro-file {targets}" - return cmd - - return { - "file_dep": [script] + list(file_deps.values()), - "actions": [create_cmd()], - "getargs": { - "size": ("get_domain_size", "size"), - "num_dofs": ("get_num_dofs", "num_dofs"), - }, - "targets": [ROOT / "macros.tex"], - "clean": True, - } - - -def task_paper(): - """compile pdf from latex source""" - paper_source = SOURCE / "paper.tex" - paper = ROOT / "paper.tex" - return { - "file_dep": [paper_source, ROOT / "macros.tex", ROOT / "plotoverline.csv"], - "actions": [f"cp {paper_source} {paper}", f"tectonic {paper}"], - "targets": [paper, ROOT / "paper.pdf"], - "clean": True, - } diff --git a/exemplary_workflow/pyiron/README.md b/exemplary_workflow/pyiron/README.md deleted file mode 100644 index 18855d4e..00000000 --- a/exemplary_workflow/pyiron/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# pyiron -This directory contains an implementation of the exemplary workflow with [pyiron](https://pyiron.org/) or more specifically the [pyiron_base](https://pyiron-base.readthedocs.io) workflow manager. - -## Installation -For more detailed information we refer to the official [documentation](https://pyiron-base.readthedocs.io/en/latest/installation.html). -The recommended way of installing `pyiron_base` is via conda, because it also enables `pyiron_base` to handle software dependencies of your workflow using the [conda_subprocess](https://github.com/pyiron/conda_subprocess) package. - -You can then install `pyiron_base` with -```sh -mamba create -c conda-forge -n pyiron_base pyiron_base=0.9.7 conda_subprocess=0.0.4 -``` - -## Running the exemplary workflow -The workflow can be run with -```sh -python workflow.py -``` -The final output `paper.pdf` created using the `tectonic` package is then stored in the working direcrtory of last workflow step: -```sh -./workflow/tectonic_hdf5/tectonic/paper.pdf -``` -This is a general feature of the `pyiron_base` workflow manager, each step of the workflow - called job - is executed in a separate working directory. diff --git a/exemplary_workflow/pyiron/workflow.py b/exemplary_workflow/pyiron/workflow.py deleted file mode 100644 index dfad4edc..00000000 --- a/exemplary_workflow/pyiron/workflow.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import shutil -from pyiron_base import Project - -# input parameter -domain_size = 2.0 - -# create pyiron project -pr = Project("workflow") - -# create conda environments for the proprocessing, processing and postprocessing stage -for k, v in { - "preprocessing": "../source/envs/preprocessing.yaml", - "processing": "../source/envs/processing.yaml", - "postprocessing": "../source/envs/postprocessing.yaml" -}.items(): - pr.conda_environment.create(env_name=k, env_file=v, global_installation=False) - - -# Preprocessing -## generate mesh -gmsh = pr.wrap_executable( - executable_str=f"gmsh -2 -setnumber domain_size {domain_size} unit_square.geo -o square.msh", - conda_environment_path=pr.conda_environment.preprocessing, - input_file_lst=["../source/unit_square.geo"], - delayed=True, - output_file_lst=["square.msh"], -) - -## convert mesh to xdmf -meshio = pr.wrap_executable( - executable_str="meshio convert square.msh square.xdmf", - conda_environment_path=pr.conda_environment.preprocessing, - input_file_lst=[gmsh.files.square_msh], - delayed=True, - output_file_lst=["square.xdmf", "square.h5"], -) - - -# Processing -## poisson -def collect_output(working_directory): - with open(os.path.join(working_directory, "numdofs.txt"), "r") as f: - return {"numdofs": int(f.read())} - -poisson = pr.wrap_executable( - executable_str="python poisson.py --mesh square.xdmf --degree 2 --outputfile poisson.pvd --num-dofs numdofs.txt", - conda_environment_path=pr.conda_environment.processing, - input_file_lst=["../source/poisson.py", meshio.files.square_xdmf, meshio.files.square_h5], - delayed=True, - collect_output_funct=collect_output, - output_key_lst=["numdofs"], - output_file_lst=["poisson.pvd", "poisson000000.vtu"], -) - - -# Postprocessing -## plot over line -pvbatch = pr.wrap_executable( - executable_str="pvbatch postprocessing.py poisson.pvd plotoverline.csv", - conda_environment_path=pr.conda_environment.postprocessing, - input_file_lst=["../source/postprocessing.py", poisson.files.poisson_pvd, poisson.files.poisson000000_vtu], - delayed=True, - output_file_lst=["plotoverline.csv"], -) - -## substitute macros -def write_input(input_dict, working_directory): - script_name = os.path.join(working_directory, "macros.sh") - with open(script_name, "w") as f: - f.writelines(f"python prepare_paper_macros.py --macro-template-file macros.tex.template --plot-data-path plotoverline.csv --domain-size {domain_size} --num-dofs {input_dict["numdofs"]} --output-macro-file macros.tex") - os.chmod(script_name, 0o744) - -macros = pr.wrap_executable( - input_dict={"numdofs": poisson.output.numdofs}, - write_input_funct=write_input, - executable_str="./macros.sh", - conda_environment_path=pr.conda_environment.postprocessing, - input_file_lst=["../source/macros.tex.template", "../source/prepare_paper_macros.py", pvbatch.files.plotoverline_csv], - delayed=True, - output_file_lst=["macros.tex"], -) - -## compile paper -tectonic = pr.wrap_executable( - executable_str="tectonic paper.tex", - conda_environment_path=pr.conda_environment.postprocessing, - input_file_lst=["../source/paper.tex", macros.files.macros_tex, pvbatch.files.plotoverline_csv], - delayed=True, - output_file_lst=["paper.pdf"], -) - -# Execute Workflow Graph and copy output -result = tectonic.pull() -shutil.copyfile(str(result.files.paper_pdf), "paper.pdf") diff --git a/exemplary_workflow/snakemake/.gitignore b/exemplary_workflow/snakemake/.gitignore deleted file mode 100644 index c217326e..00000000 --- a/exemplary_workflow/snakemake/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.snakemake -paper.pdf -preprocessing/ -processing/ -postprocessing/ diff --git a/exemplary_workflow/snakemake/README.md b/exemplary_workflow/snakemake/README.md deleted file mode 100644 index 01dda2af..00000000 --- a/exemplary_workflow/snakemake/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Snakemake -This directory contains an implementation of the exemplary workflow with [Snakemake](https://snakemake.github.io/). - -## Installation -For more detailed information we refer to the official [documentation](https://snakemake.readthedocs.io/en/stable/getting_started/installation.html). -The recommended way of installing snakemake is via conda, because it also enables Snakemake -to handle software dependencies of your workflow. -In the documentation the use of the [Mambaforge](https://github.com/conda-forge/miniforge#mambaforge) Python 3 distribution (Mambaforge is a Conda based distribution like [Miniconda](https://docs.conda.io/en/latest/miniconda.html), which however uses [Mamba](https://github.com/mamba-org/mamba) a fast and more robust replacement for the Conda package manager) is recommended. -If you don't have the Mamba command because you used a different conda distribution than Mambaforge, you can also first install Mamba in your base environment with -```sh -conda install -n base -c conda-forge mamba -``` -You can then install snakemake with -```sh -mamba create -c conda-forge -c bioconda -n snakemake snakemake -``` -If you don't want to use Mamba, install snakemake with the usual command -```sh -conda create -c conda-forge -c bioconda -n snakemake snakemake -``` - -## Running the exemplary workflow -The workflow can be run with -```sh -snakemake --cores 1 --use-conda ./paper.pdf -``` -with Mamba as the default conda frontend. -The parameter `domain_size` with default value 2.0 is defined in the configuration file `config.yaml`. -The values defined in the configuration file can be overwritten via the command line, e.g. -```sh -snakemake --cores 1 --use-conda --config domain_size=4.0 ./paper.pdf -``` -If you don't want to use Mamba, you need to specify conda as the frontend for installing environments. -This can be done by appending `--conda-frontend conda`. diff --git a/exemplary_workflow/snakemake/Snakefile b/exemplary_workflow/snakemake/Snakefile deleted file mode 100644 index ff3fe8e5..00000000 --- a/exemplary_workflow/snakemake/Snakefile +++ /dev/null @@ -1,100 +0,0 @@ -configfile: "./config.yaml" - - -# NOTE -# $ snakemake --use-conda --cores 1 --config domain_size=4.0 -# does not re-execute the job generated from rule 'generate_mesh' -# if it was executed before with e.g. the default value 2.0 -# It seems one has to force the execution of 'somerule' in this case -# $ snakemake -R somerule -rule generate_mesh: - input: - "../source/unit_square.geo", - output: - "preprocessing/square.msh", - conda: - "../source/envs/preprocessing.yaml" - log: - "logs/gmsh.log", - shell: - "gmsh -2 -setnumber domain_size {config[domain_size]} {input} -o {output} > {log} 2>&1" - - -rule convert_to_xdmf: - input: - rules.generate_mesh.output, - output: - "preprocessing/square.xdmf", - conda: - "../source/envs/preprocessing.yaml" - log: - "logs/meshio.log", - shell: - "meshio convert {input} {output} > {log} 2>&1" - - -rule poisson: - input: - py="../source/poisson.py", - xdmf=rules.convert_to_xdmf.output, - output: - pvd="processing/poisson.pvd", - txt="processing/numdofs.txt", - conda: - "../source/envs/processing.yaml" - log: - "logs/fenics.log", - shell: - "python {input.py} --mesh {input.xdmf} --degree 2 --outputfile {output.pvd} --num-dofs {output.txt} > {log} 2>&1" - - -rule plot_over_line: - input: - py="../source/postprocessing.py", - pvd=rules.poisson.output.pvd, - output: - "postprocessing/plotoverline.csv", - conda: - "../source/envs/postprocessing.yaml" - log: - "logs/paraview.log", - shell: - "pvbatch {input.py} {input.pvd} {output} > {log} 2>&1" - - -rule substitute_macros: - input: - py="../source/prepare_paper_macros.py", - template="../source/macros.tex.template", - data=rules.plot_over_line.output, - ndofs=rules.poisson.output.txt, - output: - "macros.tex", - log: - "logs/macros.log", - run: - with open(input.ndofs, "r") as instream: - ndofs = int(instream.read()) - shell( - "python {input.py} --macro-template-file {input.template} \ - --plot-data-path {input.data} \ - --domain-size {config[domain_size]} \ - --num-dofs %s \ - --output-macro-file {output[0]}" - % (ndofs) - ) - - -rule compile_paper: - input: - paper="../source/paper.tex", - macros=rules.substitute_macros.output, - output: - tex="paper.tex", - pdf="paper.pdf", - conda: - "../source/envs/postprocessing.yaml" - log: - "logs/tectonic.log", - shell: - "cp {input.paper} {output.tex} && tectonic {output.tex} > {log} 2>&1" diff --git a/exemplary_workflow/snakemake/config.yaml b/exemplary_workflow/snakemake/config.yaml deleted file mode 100644 index 2d2d71f3..00000000 --- a/exemplary_workflow/snakemake/config.yaml +++ /dev/null @@ -1 +0,0 @@ -domain_size: 2.0 diff --git a/img/aiida_provenance_graph.png b/img/aiida_provenance_graph.png deleted file mode 100644 index bf986d26..00000000 Binary files a/img/aiida_provenance_graph.png and /dev/null differ diff --git a/img/exemplary_wf_dag.png b/img/exemplary_wf_dag.png deleted file mode 100644 index a5f21426..00000000 Binary files a/img/exemplary_wf_dag.png and /dev/null differ diff --git a/img/fairness.png b/img/fairness.png deleted file mode 100644 index 79c76120..00000000 Binary files a/img/fairness.png and /dev/null differ diff --git a/img/fairness.svg b/img/fairness.svg deleted file mode 100755 index c7ecdf6f..00000000 --- a/img/fairness.svg +++ /dev/null @@ -1,2705 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - </> - - - - - Data - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Software/hardwaredependencies - - - - - - - - Workflow - - - - - - - - - - - - - - - Scientific publication - - - - - - - - Source code - - - - Metadata - - - - diff --git a/img/workflows.svg b/img/workflows.svg deleted file mode 100644 index e066e83d..00000000 --- a/img/workflows.svg +++ /dev/null @@ -1,2624 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - 100110100110110101011010100101100011000110110111 - - - - 100110100110110101011010100101100011000110110111 - - - - - - - - - - </> - - - - - - - - - - - - - - - - - - - - Workflow - - - - - - - - - Input - - - - - Output - - - - - - Process - - - - - - 100110100110110101011010100101100011000110110111 - - - - - Input - - - 100110100110110101011010100101100011000110110111 - - - - - Output - - - - - - - - - - - - - - - - - - - - - - - Source code - - - Executable - - - - - - - compiler- third-party libraries- ... - - - - - Computationenvironment - - - - diff --git a/index.rst b/index.rst deleted file mode 100644 index 47fb7232..00000000 --- a/index.rst +++ /dev/null @@ -1,56 +0,0 @@ -Scientific workflow requirements - interfaces to combine different simulation and data processing tools -======================================================================================================= - -Software-driven scientific workflows are often characterized by a complex interplay -of various pieces of software executed in a particular order. The output of one -process may serve as input to a subsequent process, which requires -them to be executed sequentially with a proper mapping of outputs to inputs. -Other computations are independent of each other and can be executed in parallel. -Thus, one of the main tasks of workflow handling is the proper and efficient scheduling -of the individual processes. - -.. image:: ./img/workflows.png - :width: 400 - :alt: TODO: caption - -Typically, each process in the workflow, just as the workflow itself, takes some -input to produce output data and the behavior of a process is determined by its -source code and/or executable binaries/packages that are called within it. Beyond -this, the computation environment not only has a significant influence on its -behavior, but is also crucial in order for the process to work at all. The environment -includes interpreters and/or compilers, as well as third-party libraries and packages -that contribute to the computations carried out in the process. The exact version -numbers of all involved packages are crucial, as the workflow may fail to work with -newer or older packages, or, may produce different results. - -In order for research workflows to comply with the -`FAIR principles `_ (originally formulated -for research data), they should be: - -- published -- documented -- annotated with metadata -- reproducible -- reusable - -Because of the dependency on their environment, reproducibility and reusability -is not trivial to achieve for scientific workflows. The results presented in a -scientific paper, for instance, are produced with a very specific environment, -and in order to guarantee reproducibility, this environment has to be reinstantiated -possibly many years later. - -In this documentation we want to elaborate, together with the scientific community, -a vision on how scientific workflows should be created/packaged/published in order -to be as FAIR as possible. We want to evaluate if existing workflow tools provide -reusable solutions and identify the capabilities that are missing to reach our goal: -reproducible research workflows, by anyone, anywhere and anytime. - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - docs/fairness.rst - docs/userstories.rst - docs/requirements.rst - docs/exemplarywf.rst - docs/tools.rst diff --git a/exemplary_workflow/source/envs/default_env.yaml b/source/envs/default_env.yaml similarity index 100% rename from exemplary_workflow/source/envs/default_env.yaml rename to source/envs/default_env.yaml diff --git a/exemplary_workflow/source/envs/full_default_env.yaml b/source/envs/full_default_env.yaml similarity index 100% rename from exemplary_workflow/source/envs/full_default_env.yaml rename to source/envs/full_default_env.yaml diff --git a/exemplary_workflow/source/envs/postprocessing.yaml b/source/envs/postprocessing.yaml similarity index 100% rename from exemplary_workflow/source/envs/postprocessing.yaml rename to source/envs/postprocessing.yaml diff --git a/exemplary_workflow/source/envs/preprocessing.yaml b/source/envs/preprocessing.yaml similarity index 100% rename from exemplary_workflow/source/envs/preprocessing.yaml rename to source/envs/preprocessing.yaml diff --git a/exemplary_workflow/source/envs/processing.yaml b/source/envs/processing.yaml similarity index 100% rename from exemplary_workflow/source/envs/processing.yaml rename to source/envs/processing.yaml diff --git a/exemplary_workflow/source/macros.tex.template b/source/macros.tex.template similarity index 100% rename from exemplary_workflow/source/macros.tex.template rename to source/macros.tex.template diff --git a/exemplary_workflow/source/paper.tex b/source/paper.tex similarity index 100% rename from exemplary_workflow/source/paper.tex rename to source/paper.tex diff --git a/exemplary_workflow/source/poisson.py b/source/poisson.py similarity index 100% rename from exemplary_workflow/source/poisson.py rename to source/poisson.py diff --git a/exemplary_workflow/source/postprocessing.py b/source/postprocessing.py similarity index 100% rename from exemplary_workflow/source/postprocessing.py rename to source/postprocessing.py diff --git a/exemplary_workflow/source/prepare_paper_macros.py b/source/prepare_paper_macros.py similarity index 100% rename from exemplary_workflow/source/prepare_paper_macros.py rename to source/prepare_paper_macros.py diff --git a/exemplary_workflow/source/unit_square.geo b/source/unit_square.geo similarity index 100% rename from exemplary_workflow/source/unit_square.geo rename to source/unit_square.geo diff --git a/workflow.json b/workflow.json new file mode 100644 index 00000000..911beb1f --- /dev/null +++ b/workflow.json @@ -0,0 +1,24 @@ +{ + "nodes": [ + {"id": 0, "function": "workflow.generate_mesh"}, + {"id": 1, "function": "workflow.convert_to_xdmf"}, + {"id": 2, "function": "workflow.poisson"}, + {"id": 3, "function": "workflow.plot_over_line"}, + {"id": 4, "function": "workflow.substitute_macros"}, + {"id": 5, "function": "workflow.compile_paper"}, + {"id": 6, "value": 2.0} + ], + "edges": [ + {"target": 0, "targetPort": "domain_size", "source": 6, "sourcePort": null}, + {"target": 1, "targetPort": "gmsh_output_file", "source": 0, "sourcePort": null}, + {"target": 2, "targetPort": "meshio_output_xdmf", "source": 1, "sourcePort": "xdmf_file"}, + {"target": 2, "targetPort": "meshio_output_h5", "source": 1, "sourcePort": "h5_file"}, + {"target": 3, "targetPort": "poisson_output_pvd_file", "source": 2, "sourcePort": "pvd_file"}, + {"target": 3, "targetPort": "poisson_output_vtu_file", "source": 2, "sourcePort": "vtu_file"}, + {"target": 4, "targetPort": "pvbatch_output_file", "source": 3, "sourcePort": null}, + {"target": 4, "targetPort": "ndofs", "source": 2, "sourcePort": "numdofs"}, + {"target": 4, "targetPort": "domain_size", "source": 6, "sourcePort": null}, + {"target": 5, "targetPort": "macros_tex", "source": 4, "sourcePort": null}, + {"target": 5, "targetPort": "plot_file", "source": 3, "sourcePort": null} + ] +} \ No newline at end of file diff --git a/workflow.py b/workflow.py new file mode 100644 index 00000000..3893a38e --- /dev/null +++ b/workflow.py @@ -0,0 +1,137 @@ +import os +from conda_subprocess import check_output +import shutil + +source_directory = os.path.join(os.path.dirname(os.path.abspath(__file__)), "source") + + +def generate_mesh(domain_size: float = 2.0) -> str: + stage_name = "preprocessing" + gmsh_output_file_name = "square.msh" + source_file_name ="unit_square.geo" + os.makedirs(stage_name, exist_ok=True) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _ = check_output( + [ + "gmsh", "-2", "-setnumber", "domain_size", str(domain_size), + source_file_name, "-o", gmsh_output_file_name + ], + prefix_name=stage_name, + cwd=stage_name, + universal_newlines=True, + ).split("\n") + return os.path.abspath(os.path.join(stage_name, gmsh_output_file_name)) + + +def convert_to_xdmf(gmsh_output_file : str) -> dict: + stage_name = "preprocessing" + meshio_output_file_name = "square.xdmf" + os.makedirs(stage_name, exist_ok=True) + _copy_file(stage_name=stage_name, source_file=gmsh_output_file) + _ = check_output( + ["meshio", "convert", os.path.basename(gmsh_output_file), meshio_output_file_name], + prefix_name=stage_name, + cwd=stage_name, + universal_newlines=True, + ).split("\n") + return { + "xdmf_file": os.path.abspath(os.path.join(stage_name, meshio_output_file_name)), + "h5_file": os.path.join(os.path.abspath(stage_name), "square.h5"), + } + + +def poisson(meshio_output_xdmf: str, meshio_output_h5: str) -> dict: + stage_name = "processing" + poisson_output_pvd_file_name = "poisson.pvd" + poisson_output_numdofs_file_name = "numdofs.txt" + source_file_name = "poisson.py" + os.makedirs(stage_name, exist_ok=True) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file(stage_name=stage_name, source_file=meshio_output_xdmf) + _copy_file(stage_name=stage_name, source_file=meshio_output_h5) + _ = check_output( + [ + "python", "poisson.py", "--mesh", os.path.basename(meshio_output_xdmf), "--degree", "2", + "--outputfile", poisson_output_pvd_file_name, "--num-dofs", poisson_output_numdofs_file_name + ], + prefix_name=stage_name, + cwd=stage_name, + universal_newlines=True, + ).split("\n") + return { + "numdofs": _poisson_collect_output(numdofs_file=os.path.join(stage_name, poisson_output_numdofs_file_name)), + "pvd_file": os.path.abspath(os.path.join(stage_name, poisson_output_pvd_file_name)), + "vtu_file": os.path.abspath(os.path.join(stage_name, "poisson000000.vtu")), + } + + +def plot_over_line(poisson_output_pvd_file: str, poisson_output_vtu_file: str) -> str: + stage_name = "postprocessing" + pvbatch_output_file_name = "plotoverline.csv" + source_file_name = "postprocessing.py" + os.makedirs(stage_name, exist_ok=True) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file(stage_name=stage_name, source_file=poisson_output_pvd_file) + _copy_file(stage_name=stage_name, source_file=poisson_output_vtu_file) + _ = check_output( + ["pvbatch", source_file_name, os.path.basename(poisson_output_pvd_file), pvbatch_output_file_name], + prefix_name=stage_name, + cwd=stage_name, + universal_newlines=True, + ).split("\n") + return os.path.abspath(os.path.join("postprocessing", pvbatch_output_file_name)) + + +def substitute_macros(pvbatch_output_file: str, ndofs: int, domain_size: float = 2.0) -> str: + stage_name = "postprocessing" + source_file_name = "prepare_paper_macros.py" + template_file_name = "macros.tex.template" + macros_output_file_name = "macros.tex" + os.makedirs(stage_name, exist_ok=True) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file_from_source(stage_name=stage_name, source_file_name=template_file_name) + _copy_file(stage_name=stage_name, source_file=pvbatch_output_file) + _ = check_output( + [ + "python", "prepare_paper_macros.py", "--macro-template-file", template_file_name, + "--plot-data-path", os.path.basename(pvbatch_output_file), "--domain-size", str(domain_size), + "--num-dofs", str(ndofs), "--output-macro-file", macros_output_file_name, + ], + prefix_name=stage_name, + cwd=stage_name, + universal_newlines=True, + ).split("\n") + return os.path.abspath(os.path.join(stage_name, macros_output_file_name)) + + +def compile_paper(macros_tex: str, plot_file: str) -> str: + stage_name = "postprocessing" + paper_output = "paper.pdf" + source_file_name = "paper.tex" + os.makedirs(stage_name, exist_ok=True) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file(stage_name=stage_name, source_file=macros_tex) + _copy_file(stage_name=stage_name, source_file=plot_file) + _ = check_output( + ["tectonic", source_file_name], + prefix_name=stage_name, + universal_newlines=True, + cwd=stage_name, + ).split("\n") + return os.path.abspath(os.path.join(stage_name, paper_output)) + + +def _poisson_collect_output(numdofs_file: str) -> int: + with open(os.path.join(numdofs_file), "r") as f: + return int(f.read()) + + +def _copy_file(stage_name, source_file): + input_file = os.path.join(os.path.abspath(stage_name), os.path.basename(source_file)) + if input_file != source_file: + shutil.copyfile(source_file, input_file) + + +def _copy_file_from_source(stage_name, source_file_name): + source_file = os.path.join(source_directory, source_file_name) + shutil.copyfile(source_file, os.path.join(stage_name, source_file_name)) \ No newline at end of file