Skip to content

Commit

Permalink
fix(executor): override default resources to remove mem/disk (reanahu…
Browse files Browse the repository at this point in the history
  • Loading branch information
mdonadoni committed Mar 22, 2024
1 parent b0e3669 commit ec1ec20
Showing 1 changed file with 19 additions and 11 deletions.
30 changes: 19 additions & 11 deletions reana_workflow_engine_snakemake/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from snakemake.common import async_lock
from snakemake.executors import ClusterExecutor, GenericClusterExecutor
from snakemake.jobs import Job
from snakemake.resources import DefaultResources
from snakemake import scheduler # for monkeypatch

from reana_workflow_engine_snakemake.config import (
Expand Down Expand Up @@ -250,15 +251,26 @@ def run_jobs(
operational_options={},
):
"""Run Snakemake jobs using custom REANA executor."""
workflow_file_path = os.path.join(workflow_workspace, workflow_file)
common_snakemake_args = dict(
snakefile=workflow_file_path,
config=workflow_parameters,
workdir=workflow_workspace,
keep_logger=True,
# Since Snakemake v7.3.0, the workflow logs include Snakemake-percieved native
# resource information on memory and storage (`mem_mb`, `disk_mb`) for each
# Snakemake rule run on cloud. However, REANA overrides these when running
# user jobs, so we should hide these in order not to present any misleading
# information to users. For this reason, the default resources are overridden
# here with the only the "bare" ones (`tmpdir`).
default_resources=DefaultResources(mode="bare"),
)

def _generate_report(workflow_file_path):
def _generate_report():
"""Generate HTML report."""
success = snakemake(
workflow_file_path,
config=workflow_parameters,
workdir=workflow_workspace,
**common_snakemake_args,
report=operational_options.get("report", DEFAULT_SNAKEMAKE_REPORT_FILENAME),
keep_logger=True,
)
if not success:
log.error("Error generating workflow HTML report.")
Expand All @@ -269,21 +281,17 @@ def _generate_report(workflow_file_path):
# Monkeypatch GenericClusterExecutor class in `scheduler` module
scheduler.GenericClusterExecutor = REANAClusterExecutor

workflow_file_path = os.path.join(workflow_workspace, workflow_file)
success = snakemake(
workflow_file_path,
**common_snakemake_args,
printshellcmds=True,
# FIXME: Can be anything as it's not directly used. It's supposed
# to be the shell command to submit to job e.g. `condor_q`,
# but we call RJC API client instead.
cluster="reana",
config=workflow_parameters,
workdir=workflow_workspace,
notemp=True,
nodes=SNAKEMAKE_MAX_PARALLEL_JOBS, # enables DAG parallelization
keep_logger=True,
)
# Once the workflow is finished, generate the report,
# taking into account the metadata generated.
_generate_report(workflow_file_path)
_generate_report()
return success

0 comments on commit ec1ec20

Please sign in to comment.