-
Notifications
You must be signed in to change notification settings - Fork 51
/
torque-lido.tmpl
80 lines (69 loc) · 3.2 KB
/
torque-lido.tmpl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
#!/bin/bash
<%
## Check some resources and set sane defaults
resources$walltime = asInt(resources$walltime, lower = 60L, upper = 3600L * 672L)
resources$memory = asInt(resources$memory, lower = 100L, upper = 64L * 1024L)
resources$ncpus = if (is.null(resources$ncpus)) 1L else asInt(resources$ncpus, lower = 1L)
resources$modules = if (is.null(resources$modules)) character(0L) else assertCharacter(resources$modules, any.missing = FALSE)
resources$R = if (is.null(resources$R)) "R/3.4.1-gcc49-base" else assertString(resources$R)
resources$omp.threads = if (is.null(resources$omp.threads)) 1L else asInt(resources$omp.threads, lower = 1L)
resources$blas.threads = if (is.null(resources$blas.threads)) 1L else asInt(resources$blas.threads, lower = 1L)
if (!is.null(resources$type)) assertString(resources$type)
if (resources$memory > 15000)
resources$type = "quad"
use.mpi = (resources$pm.backend %??% "default") == "mpi"
## first string of queue, selected by walltime
walltimes = 3600L * c(1L, 8L, 48L, 672L)
queue = c("short", "med", "long", "ultralong")[wf(resources$walltime <= walltimes)]
## check default modules
modules.default = c(binutils = "binutils/2.25", gcc = "gcc/4.9.3", openblas = "openblas/0.2.17")
modules = resources$modules
for (i in seq_along(modules.default)) {
if (!any(grepl(paste0("^", names(modules.default[i]), "/?[0-9\\.]*$"), modules))) {
modules = paste(modules, modules.default[i])
}
}
## add R
modules = paste(modules, resources$R)
## add mpi
if (use.mpi)
modules = paste(modules, "openmpi/gcc4.9.x")
## very ugly hack because we cannot log to data (nobackup) filesystem on lido,
## only home fs is available
## unfortunately there seems to be no generic solution
## does log path start with /data/?
log.file = log.file
if (length(grep("^/data/", log.file)) > 0L) {
## strip that
log.file = substr(log.file, 7L, nchar(log.file))
## find next forward slash
i = regexpr("/", log.file)
if (i != -1) {
## this must be "user": e.g. /data/bischl/...
user = substr(log.file, 1L, i-1L)
## put together
log.file = sprintf("/home/%s/nobackup%s", user, substr(log.file, i, nchar(log.file)))
}
}
-%>
#PBS -N <%= job.name %>
#PBS -o <%= log.file %>
#PBS -l walltime=<%= resources$walltime %>,nodes=<%= if (use.mpi) resources$ncpus else 1 %>:ppn=<%= if (use.mpi) 1 else resources$ncpus %><%= if (!is.null(resources$type)) paste0(":", resources$type) %>,vmem=<%= resources$memory %>M
#PBS -q <%= queue %>
#PBS -j oe
<%= if (array.jobs) sprintf("#PBS -t 1-%i", nrow(jobs)) else "" %>
## setup modules
source /sysdata/shared/sfw/Modules/default/init/bash
module add <%= modules %>
## create our own temp dir (and clean it up later), lido does not do this automatically
mkdir /scratch/${USER}-${PBS_JOBID}
export TMPDIR=/scratch/${USER}-${PBS_JOBID}
<%= sprintf("export OMP_NUM_THREADS=%i", resources$omp.threads) -%>
<%= sprintf("export OPENBLAS_NUM_THREADS=%i", resources$blas.threads) -%>
<%= sprintf("export MKL_NUM_THREADS=%i", resources$blas.threads) -%>
## export value of DEBUGME environemnt var to slave
export DEBUGME=<%= Sys.getenv("DEBUGME") %>
## run R
<%= if (use.mpi) "mpirun -np 1 " else "" %>Rscript -e 'batchtools::doJobCollection("<%= uri %>")'
## Cleanup
rm -rf /scratch/${USER}-${PBS_JOBID}