-
Notifications
You must be signed in to change notification settings - Fork 0
/
objective.py
executable file
·324 lines (294 loc) · 11.3 KB
/
objective.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
"""
objective function utilized by hyperopt-monogdb-worker
Receives hyperparameters, reformats them as a nwt,
runs the model, pulls trial information from .list
file, computes loss, and returns that loss
"""
# Disabling pylint snake_case warnings, import error warnings, and
# redefining out of scope warnings, too many local variables
# too many branches, too many statements
#
# pylint: disable = E0401, C0103, W0621, R0914, R0912, R0915, R0915
import os
import math
from datetime import datetime
from subprocess import run, TimeoutExpired
from shutil import copyfile
import pandas as pd
from hyperopt import STATUS_OK
def inputHp2nwt(inputHp, cwd):
"""
Takes Hyperopt Hyperparameter format and reformats it as a .nwt file. This .nwt file
overwrites the local .nwt file and will thus be used by MODFLOW
inputHp - hyperopt hyperparams
cwd - working directory of project
creates *.nwt file
returns path to *.nwt file
"""
# keep track of which NWTNUM the machine is on
with open(os.path.join(cwd, 'nwts', 'nwtnum.txt'), 'r+') as f:
NWTNUM = int(f.read())
f.seek(0)
f.truncate()
f.write(str(NWTNUM+1))
# Write the standard first line of the .nwt file
with open(os.path.join(cwd, 'nwts', ('nwt_{}.nwt'.format(NWTNUM))), 'w') as file:
file.write(('{} {} {} {} {} {} {} {} CONTINUE {} {} {} {} {} {} {} {}'.format(inputHp[1],
inputHp[2],
int(inputHp[3]),
inputHp[4],
inputHp[0]['linmeth'],
inputHp[5],
inputHp[6],
inputHp[7],
inputHp[8],
inputHp[9],
inputHp[10],
inputHp[11],
inputHp[12],
int(inputHp[13]),
inputHp[14],
inputHp[15])) + '\n')
# depending on the linmeth setting, change the formatting of the rest of the file
if inputHp[0]['linmeth'] == 1:
with open(os.path.join(cwd, 'nwts', ('nwt_{}.nwt'.format(NWTNUM))), 'a') as file:
file.write(('{} {} {} {} {}'.format(int(inputHp[0]['maxitinner']),
inputHp[0]['ilumethod'],
int(inputHp[0]['levfill']),
inputHp[0]['stoptol'],
int(inputHp[0]['msdr']))))
elif inputHp[0]['linmeth'] == 2:
with open(os.path.join(cwd, 'nwts', ('nwt_{}.nwt'.format(NWTNUM))), 'a') as file:
file.write(('{} {} {} {} {} {} {} {} {} {}'.format(inputHp[0]['iacl'],
inputHp[0]['norder'],
int(inputHp[0]['level']),
int(inputHp[0]['north']),
inputHp[0]['iredsys'],
inputHp[0]['rrctols'],
inputHp[0]['idroptol'],
inputHp[0]['epsrn'],
inputHp[0]['hclosexmd'],
int(inputHp[0]['mxiterxmd']))))
return os.path.join(cwd, 'nwts', ('nwt_{}.nwt'.format(NWTNUM)))
def trials2csv(trials, d):
"""
Converts Trials Dataframe Object to csv
trials - MongoTrials object
d - directory to save to
outputs nwt_performance.csv
"""
df = pd.DataFrame(trials.results).drop('loss', axis=1)
df.to_csv(os.path.join(d, 'nwt_performance.csv'))
def getModelRunCommands(cwd):
"""
Pull time limit and run command from run.sh file
cwd - directory containing run.sh
returns time limit, run command
"""
last_line = ""
run_command = ""
use_next = False
# open run.sh and look for run command and last line
with open(os.path.join(cwd, 'run.sh')) as f:
for line in f:
if use_next:
run_command = line
use_next = False
elif line.startswith('# Run Command:'):
use_next = True
last_line = line
# last line should be timout, if empty or non-convertable timelim none
try:
timelim = float(last_line) * 60
print(f'[INFO] Timeout for model run is set to {timelim / 60} minutes')
except ValueError:
timelim = None
print('[INFO] No timeout set for model run')
return timelim, run_command
def runModel(pathtonwt, initnwt, cwd, timelim, run_command):
"""
Run the MODFLOW Model/Run Command using subprocess.run
pathtonwt - path to generated nwt
initnwt - path to initial nwt
cwd - working directory of the project
timelim - time limit
run_command - run command located in run.sh
returns True if successful terminimation of trial,
else returns false due to TimeoutExpired error
* NOTE * "successful termination" doesn't necessarily mean the model
didn't error out, only that the process has finished.
"""
# replace initial nwt with newly generated nwt
copyfile(pathtonwt, os.path.join(cwd, initnwt))
print(f'[INFO] Using run command: {run_command.strip()}')
print(f'[INFO] Starting run out of {cwd}')
# try running, and if timeout catch
try:
modflowProcess = run(run_command.strip().split(' '),
cwd = cwd,
capture_output = True,
timeout = timelim,
check = True)
print(str(modflowProcess.stdout, 'utf-8'), '\n', str(modflowProcess.stderr, 'utf-8'))
print("[INFO] Successful termination of trial")
return True
except TimeoutExpired:
print('[WARNING] Time Limit reached, terminating run')
return False
def getRunResults(cwd, listfile):
"""
Pull Run Results from MODFLOW *.list file
cwd - project running directory
listfile - path to *.list file
if successful:
returns sec_elapsed, iterations, mass_balance
else:
returns 999999, -1, 999999
"""
# find all the necessary lines in the .list file
mbline, timeline, iterline = '', '', ''
with open(os.path.join(cwd, listfile), 'r') as file:
mbfound = False
for line in reversed(list(file)):
if 'Error in Preconditioning' in line:
return 999999, -1, 999999
if 'PERCENT DISCREPANCY' in line and mbfound is False:
mbfound = True
mbline = line
if 'Elapsed run time' in line:
timeline = line
if 'OUTER ITERATIONS' in line:
iterline = line
break
# check for run failure
if timeline == '':
return 999999, -1, 999999
mass_balance = None
# pull mass balance
for val in mbline.split(' '):
try:
mass_balance = float(val)
break
except ValueError:
pass
if not mass_balance:
print('[ERROR] bad run')
return 999999, -1, 999999
# prepare to pull run time information
foundmin, foundsec, foundhour = False, False, False
minutes, sec, hrs, days = 0, 0, 0, 0
# MODFLOW doesn't report timing consitantly, so we have
# to use some weird looping and calculations to
# pull the correct time information
for val in reversed(timeline.split(' ')):
if foundsec is False:
try:
sec = float(val)
foundsec = True
except ValueError:
pass
elif foundmin is False:
try:
minutes = float(val)
foundmin = True
except ValueError:
pass
elif foundhour is False:
try:
hrs = float(val)
foundhour = True
except ValueError:
pass
else:
try:
days = float(val)
break
except ValueError:
pass
# calculate how long everything took
sec_elapsed = days * 24 * 3600 + hrs * 3600 + minutes * 60 + sec
# check for good values
if sec_elapsed == 0:
print('[ERROR] bad run')
return 999999, -1, 999999
iterations = None
for val in iterline.split(' '):
try:
iterations = float(val)
break
except ValueError:
pass
if not iterations:
print('[ERROR] bad run')
return 999999, -1, 999999
print('[MASS BALANCE]:', mass_balance)
print('[SECONDS]:', sec_elapsed)
print('[TOTAL ITERATIONS]:', iterations)
return sec_elapsed, iterations, mass_balance
def objective(inputHp):
"""
Objective funciton that is called by hyperopt-mongo-worker
inputHp - hyperparams from Hyperopt
returns dictionary of trial run metrics:
{'loss': loss,
'status': STATUS_OK,
'eval_time': eval_time,
'mass_balance': mass_balance,
'sec_elapsed': sec_elapsed,
'iterations': iterations,
'NWT Used': pathtonwt,
'finish_time': finish_time}
"""
# get eval time, set up main variables to run
eval_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
cwd = os.path.join(os.path.dirname(os.getcwd()), 'nwtenv','bin','NWT_SUBMIT','PROJECT_FILES')
# get necessary file names and paths
for file in os.listdir(cwd):
if file.endswith('.nam'):
namefile = file
elif file.endswith('.list') or file.endswith('.lst'):
listfile = file
elif file.endswith('.nwt'):
initnwt = file
foundList, foundNWT = False, False
# pull correct list and nwt names from .name file
with open(os.path.join(cwd, namefile), 'r') as f:
while not(foundList and foundNWT):
line = f.readline()
for e in line.split(' '):
if '.list' in e or '.lst' in e:
foundList = True
listfile = e.strip()
elif '.nwt' in e:
foundNWT = True
initnwt = e.strip()
# convert hyperparams to nwt and get path
pathtonwt = inputHp2nwt(inputHp, cwd)
timelim, run_command = getModelRunCommands(cwd)
# run the model and check for errors
if not runModel(pathtonwt, initnwt, cwd, timelim, run_command):
finish_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
return {'loss': 999999999999,
'status': STATUS_OK,
'eval_time': eval_time,
'mass_balance': 999999,
'sec_elapsed': timelim,
'iterations': -1,
'NWT Used': pathtonwt,
'finish_time': finish_time}
# if no errors get run results
sec_elapsed, iterations, mass_balance = getRunResults(cwd, listfile)
if mass_balance == 999999:
loss = 999999999999
else:
loss = math.exp(mass_balance ** 2) * sec_elapsed
finish_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# do final reporting
return {'loss': loss,
'status': STATUS_OK,
'eval_time': eval_time,
'mass_balance': mass_balance,
'sec_elapsed': sec_elapsed,
'iterations': iterations,
'NWT Used': pathtonwt,
'finish_time': finish_time}