Skip to content

Commit

Permalink
feat(manager): pass custom env variables to workflow engines (#571)
Browse files Browse the repository at this point in the history
  • Loading branch information
mdonadoni committed Feb 28, 2024
1 parent 08ab9a3 commit cb9369b
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 7 deletions.
29 changes: 28 additions & 1 deletion reana_workflow_controller/config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# This file is part of REANA.
# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023 CERN.
# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024 CERN.
#
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
Expand All @@ -16,6 +16,12 @@

from reana_workflow_controller.version import __version__


def _env_vars_dict_to_k8s_list(env_vars):
"""Convert env vars stored as a dictionary into a k8s-compatible list."""
return [{"name": name, "value": str(value)} for name, value in env_vars.items()]


SQLALCHEMY_TRACK_MODIFICATIONS = False
"""Track modifications flag."""

Expand Down Expand Up @@ -117,6 +123,27 @@
]
"""Common to all workflow engines environment variables."""


WORKFLOW_ENGINE_CWL_ENV_VARS = _env_vars_dict_to_k8s_list(
json.loads(os.getenv("REANA_WORKFLOW_ENGINE_CWL_ENV_VARS", "{}"))
)
"""Environment variables to be passed to the CWL workflow engine container."""

WORKFLOW_ENGINE_SERIAL_ENV_VARS = _env_vars_dict_to_k8s_list(
json.loads(os.getenv("REANA_WORKFLOW_ENGINE_SERIAL_ENV_VARS", "{}"))
)
"""Environment variables to be passed to the serial workflow engine container."""

WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS = _env_vars_dict_to_k8s_list(
json.loads(os.getenv("REANA_WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS", "{}"))
)
"""Environment variables to be passed to the Snakemake workflow engine container."""

WORKFLOW_ENGINE_YADAGE_ENV_VARS = _env_vars_dict_to_k8s_list(
json.loads(os.getenv("REANA_WORKFLOW_ENGINE_YADAGE_ENV_VARS", "{}"))
)
"""Environment variables to be passed to the Yadage workflow engine container."""

DEBUG_ENV_VARS = (
{
"name": "WDB_SOCKET_SERVER",
Expand Down
21 changes: 15 additions & 6 deletions reana_workflow_controller/workflow_run_manager.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
# This file is part of REANA.
# Copyright (C) 2019, 2020, 2021, 2022, 2023 CERN.
# Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024 CERN.
#
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""Workflow run manager interface."""
import base64
import copy
import json
import logging
import os
Expand Down Expand Up @@ -71,6 +72,10 @@
REANA_WORKFLOW_ENGINE_IMAGE_YADAGE,
WORKFLOW_ENGINE_COMMON_ENV_VARS,
DEBUG_ENV_VARS,
WORKFLOW_ENGINE_CWL_ENV_VARS,
WORKFLOW_ENGINE_SERIAL_ENV_VARS,
WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS,
WORKFLOW_ENGINE_YADAGE_ENV_VARS,
)


Expand All @@ -92,7 +97,8 @@ class WorkflowRunManager:
"--workflow-parameters '{parameters}' "
"--operational-options '{options}' "
),
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+ WORKFLOW_ENGINE_CWL_ENV_VARS,
},
"yadage": {
"image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_YADAGE),
Expand All @@ -105,7 +111,8 @@ class WorkflowRunManager:
"--workflow-parameters '{parameters}' "
"--operational-options '{options}' "
),
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+ WORKFLOW_ENGINE_YADAGE_ENV_VARS,
},
"serial": {
"image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_SERIAL),
Expand All @@ -117,7 +124,8 @@ class WorkflowRunManager:
"--workflow-parameters '{parameters}' "
"--operational-options '{options}' "
),
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+ WORKFLOW_ENGINE_SERIAL_ENV_VARS,
},
"snakemake": {
"image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_SNAKEMAKE),
Expand All @@ -129,7 +137,8 @@ class WorkflowRunManager:
"--workflow-parameters '{parameters}' "
"--operational-options '{options}' "
),
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
"environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+ WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS,
},
}
"""Mapping between engines and their basis configuration."""
Expand Down Expand Up @@ -220,7 +229,7 @@ def retrieve_required_cvmfs_repos(self):

def _workflow_engine_env_vars(self):
"""Return necessary environment variables for the workflow engine."""
env_vars = list(
env_vars = copy.deepcopy(
WorkflowRunManager.engine_mapping[self.workflow.type_][
"environment_variables"
]
Expand Down

0 comments on commit cb9369b

Please sign in to comment.