This repository has been archived by the owner on Aug 4, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 201
/
dask-cluster.py
76 lines (63 loc) · 1.96 KB
/
dask-cluster.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import subprocess
dask_conf_path = "../utils/dask.conf"
with open(dask_conf_path, "r") as file:
dask_conf = file.read()
_dask_conf = dask_conf.split("\n")
dask_conf = list()
for i, line in enumerate(_dask_conf):
line = line.split()
if 0 < len(line):
dask_conf.append(line)
cmd = "bash ../utils/dask-setup.sh 0"
print(cmd)
process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
output, error = process.communicate()
cmd = "hostname --all-ip-addresses"
process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
output, error = process.communicate()
IPADDR = str(output.decode()).split()[0]
ENVNAME = None
ARCH = None
NWORKERS = None
DASK_SCHED_PORT = None
DASK_SCHED_BOKEH_PORT = None
DASK_WORKER_BOKEH_PORT = None
MASTER_IPADDR = None
WHOAMI = None
LOG = None
for line in dask_conf:
if line[0] == "ENVNAME":
ENVNAME = line[1]
if line[0] == "ARCH":
ARCH = line[1]
if line[0] == "NWORKERS":
NWORKERS = line[1]
if line[0] == "DASK_SCHED_PORT":
DASK_SCHED_PORT = line[1]
if line[0] == "DASK_SCHED_BOKEH_PORT":
DASK_SCHED_BOKEH_PORT = line[1]
if line[0] == "DASK_WORKER_BOKEH_PORT":
DASK_WORKER_BOKEH_PORT = line[1]
if line[1] == "MASTER":
MASTER_IPADDR = line[0]
if line[0] == IPADDR:
WHOAMI = line[1]
if line[0] == "LOG":
LOG = line[1]
cmd = "bash ../utils/dask-setup.sh " + str(ENVNAME)
cmd = cmd + " " + str(ARCH)
cmd = cmd + " " + str(NWORKERS)
cmd = cmd + " " + str(DASK_SCHED_PORT)
cmd = cmd + " " + str(DASK_SCHED_BOKEH_PORT)
cmd = cmd + " " + str(DASK_WORKER_BOKEH_PORT)
cmd = cmd + " " + str(MASTER_IPADDR)
cmd = cmd + " " + str(WHOAMI)
if LOG != None:
cmd = cmd + " " + str(LOG)
print(cmd)
process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
output, error = process.communicate()
cmd = "screen -list"
process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
output, error = process.communicate()
print(output.decode())