Skip to content

Commit

Permalink
Make batch job runnables a list instead of single string
Browse files Browse the repository at this point in the history
This way toolkit users can deploy batch jobs with multiple runnables.
This can be convenient, for example, when the user wants to install
something wih an initial "apt install" or "yum install" before running
the "real" workload.
  • Loading branch information
aaronegolden committed Apr 30, 2024
1 parent 112293c commit 5f39ac4
Show file tree
Hide file tree
Showing 5 changed files with 23 additions and 11 deletions.
4 changes: 3 additions & 1 deletion examples/serverless-batch-mpi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,9 @@ deployment_groups:
source: modules/scheduler/batch-job-template
use: [network1, sharefs]
settings:
runnable: . /share/wrfv3/submit_wrfv3.sh;
runnables:
- id: run WRF
script: . /share/wrfv3/submit_wrfv3.sh;
machine_type: c2-standard-60
task_count: 2
mpi_mode: true
Expand Down
8 changes: 5 additions & 3 deletions examples/serverless-batch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,11 @@ deployment_groups:
source: modules/scheduler/batch-job-template
use: [network1, appfs]
settings:
runnable: |
mkdir -p /sw/$BATCH_JOB_UID
echo "Hello World from task $BATCH_TASK_INDEX" > /sw/$BATCH_JOB_UID/$BATCH_TASK_INDEX.txt
runnables:
- id: step 1
script: |
mkdir -p /sw/$BATCH_JOB_UID
echo "Hello World from task $BATCH_TASK_INDEX" > /sw/$BATCH_JOB_UID/$BATCH_TASK_INDEX.txt
machine_type: n2-standard-4
task_count: 8
task_count_per_node: 4
Expand Down
2 changes: 1 addition & 1 deletion modules/scheduler/batch-job-template/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ locals {
"${path.module}/templates/batch-job-base.yaml.tftpl",
{
synchronized = var.mpi_mode
runnable = var.runnable
runnables = var.runnables
task_count = var.task_count
tasks_per_node = local.tasks_per_node
require_hosts_file = var.mpi_mode
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ taskGroups:
- barrier:
name: "wait-for-node-startup"
%{~ endif ~}
%{~ for runnable in runnables ~}
- script:
text: ${indent(12, chomp(yamlencode(runnable)))}
text: ${indent(12, chomp(yamlencode(runnable.script)))}
%{~ if synchronized ~}
- barrier:
name: "wait-for-workload-to-complete"
name: "wait-for-script-to-complete"
%{~ endif ~}
%{~ endfor ~}
%{~ if length(nfs_volumes) > 0 ~}
volumes:
%{~ for index, vol in nfs_volumes ~}
Expand Down
14 changes: 10 additions & 4 deletions modules/scheduler/batch-job-template/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,16 @@ variable "log_policy" {
}
}

variable "runnable" {
description = "A string to be executed as the main workload of the Google Cloud Batch job. This will be used to populate the generated template."
type = string
default = "## Add your workload here"
variable "runnables" {
description = "A list of shell scripts to be executed in sequence as the main workload of the Google Cloud Batch job. These will be used to populate the generated template."
type = list(object({
script = string
}))
default = [
{
script: "## Add your workload here"
}
]
}

variable "instance_template" {
Expand Down

0 comments on commit 5f39ac4

Please sign in to comment.