-
Notifications
You must be signed in to change notification settings - Fork 3
/
config.py
114 lines (106 loc) · 4.37 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
from parsl.executors import HighThroughputExecutor
from parsl.providers import CobaltProvider, AdHocProvider
from parsl.addresses import address_by_hostname
from parsl.launchers import AprunLauncher
from parsl.channels import SSHChannel
from parsl import Config
def theta_debug_and_lambda(log_dir: str) -> Config:
"""Configuration where simulation tasks run on Theta and ML tasks run on Lambda.
Args:
log_dir: Path to store monitoring DB and parsl logs
Returns:
(Config) Parsl configuration
"""
# Set a Theta config for using the KNL nodes with a single worker per node
config = Config(
run_dir=log_dir,
retries=1,
executors=[
HighThroughputExecutor(
label='knl',
max_workers=1,
address=address_by_hostname(),
provider=CobaltProvider(
queue='debug-flat-quad', # Flat has lower utilization, even though xTB is (slightly) faster on cache
account='CSC249ADCD08',
launcher=AprunLauncher(overrides="-d 256 --cc depth -j 4"),
worker_init='''
module load miniconda-3
source activate /lus/theta-fs0/projects/CSC249ADCD08/multi-site-campaigns/env-parsl
which python
''', # Active the environment
nodes_per_block=8,
init_blocks=0,
min_blocks=0,
max_blocks=1,
cmd_timeout=300,
walltime='00:60:00',
scheduler_options='#COBALT --attrs enable_ssh=1:filesystems=theta-fs0,home',
)),
HighThroughputExecutor(
address='localhost',
label="v100",
available_accelerators=8,
worker_ports=(54928, 54875), # Hard coded to match up with SSH tunnels
worker_logdir_root='/lambda_stor/homes/lward/multi-site-campaigns/parsl-run/logs',
provider=AdHocProvider(
channels=[SSHChannel('lambda2.cels.anl.gov', script_dir='/lambda_stor/homes/lward/multi-site-campaigns/parsl-run')],
worker_init='''
# Activate conda environment
source /homes/lward/miniconda3/bin/activate /home/lward/multi-site-campaigns/parsl-logs
which python
''',
),
)]
)
return config
def theta_debug_and_venti(log_dir: str) -> Config:
"""Configuration where simulation tasks run on Theta and ML tasks run on Lambda.
Args:
log_dir: Path to store monitoring DB and parsl logs
Returns:
(Config) Parsl configuration
"""
# Set a Theta config for using the KNL nodes with a single worker per node
config = Config(
run_dir=log_dir,
retries=1,
executors=[
HighThroughputExecutor(
label='cpu',
max_workers=1,
address=address_by_hostname(),
provider=CobaltProvider(
queue='debug-flat-quad', # Flat has lower utilization, even though xTB is (slightly) faster on cache
account='CSC249ADCD08',
launcher=AprunLauncher(overrides="-d 256 --cc depth -j 4"),
worker_init='''
module load miniconda-3
source activate /lus/theta-fs0/projects/CSC249ADCD08/multi-site-campaigns/env
which python
''', # Active the environment
nodes_per_block=8,
init_blocks=0,
min_blocks=0,
max_blocks=1,
cmd_timeout=300,
walltime='00:60:00',
scheduler_options='#COBALT --attrs enable_ssh=1:filesystems=theta-fs0,home',
)),
HighThroughputExecutor(
address='localhost',
label="gpu",
available_accelerators=20,
worker_ports=(54928, 54875), # Hard coded to match up with SSH tunnels
worker_logdir_root='/home/lward/multi-site-campaigns/parsl-logs',
provider=AdHocProvider(
channels=[SSHChannel('lambda5.cels.anl.gov', script_dir='/home/lward/multi-site-campaigns/parsl-logs')],
worker_init='''
# Activate conda environment
source /homes/lward/miniconda3/bin/activate /home/lward/multi-site-campaigns/env
which python
''',
),
)]
)
return config