-
Notifications
You must be signed in to change notification settings - Fork 8
/
circuit_searcher.py
executable file
·376 lines (279 loc) · 12.6 KB
/
circuit_searcher.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
#!/usr/bin/env python
import os
import time
import numpy as np
import threading
import pickle
from Submitter import CircuitSubmitter
from CircuitQuantifier import CircuitCritic, CircuitValidator
from DatabaseHandler import DatabaseHandler
from Designers import CircuitDesigner
from TaskSets import CalculationTaskSet, FilteringTaskSet
from Utilities import Settings
from Utilities import defaults
from Utilities.decorators import thread
#====================================================
def _report_times(file_name, start, end):
content = open(file_name, 'a')
content.write('%.5f\n' % (end - start))
content.close()
#====================================================
class CircuitSearcher(object):
"""
API for SCILLA functionalities.
"""
def __init__(self, circuit_params = None, general_params = None, database_path = None, settings = None):
if settings is None:
self.settings = Settings(defaults.SETTINGS)
if not os.path.isdir(self.settings.general.scratch_dir):
os.mkdir(self.settings.general.scratch_dir)
self.circuit_params = circuit_params
self.general_params = general_params
self.task_sets = []
self.db_handler = DatabaseHandler(self.settings.databases, database_path)
self.circuit_submitter = CircuitSubmitter(self.settings.general, self.general_params)
self.circuit_validator = CircuitValidator()
self.circuit_critic = CircuitCritic(self.circuit_params)
self.circuit_designer = CircuitDesigner(self.settings.general, self.circuit_params)
def add_task(self, name = 'task0',
designer = 'random_search', designer_options = {'max_iters': 10, 'max_concurrent': np.inf},
merit = 'DoubleWell', merit_options = {},
observations = [], use_library = False,
computing_resource = 'local', computing_options = {}):
# copy settings from kwargs
settings = {}
for key, value in locals().items():
if key in ['self', 'settings']: continue
settings[key] = value
# add task set based on defined designer
if designer in ['random', 'particle_swarms', 'phoenics','grid','CMAES','LBFGS', 'scipy']:
task_set = CalculationTaskSet(settings)
self.circuit_designer.add_designer(name, designer, designer_options)
self.circuit_submitter.add_submitter(computing_resource)
elif designer in ['filter_db']:
task_set = FilteringTaskSet(settings)
else:
raise NotImplementedError
self.task_sets.append(task_set)
return task_set
def _run_calculation(self, task_set):
# fetch task_set_id for easy access
task_set_id = task_set.task_set_id
# generate all primary tasks (i.e. primers for optimization iterations)
all_tasks = task_set.generate_all_tasks()
task_ids = [task['task_id'] for task in task_set.generated_tasks]
self.db_handler.refresh()
self.db_handler.add_tasks(all_tasks)
# check if we build on prior results
if task_set.settings['use_library']:
prior_observations = self.db_handler.get_prior_circuit_evaluations()
# check abortion criteria
task_set_completed = self.db_handler.task_set_completed(task_set_id)
designer_terminated = self.circuit_designer.designer_terminated(task_set)
total_start = time.time()
# enter task execution loop
start_time = time.time()
iteration = 0
while not task_set_completed or not designer_terminated:
tic = time.time()
reported_times, reported_labels = [], []
# [x] fetch all tasks remaining for this task_set
start = time.time()
remaining_tasks = self.db_handler.fetch_remaining_tasks(task_set_id)
end = time.time()
reported_times.append(end - start)
reported_labels.append('fetching remaining tasks')
# [x] force designer to generate new parameters
start = time.time()
if not task_set_completed and designer_terminated:
num_from_optimizer = 0
for remaining_task in remaining_tasks:
if remaining_task['from_optimizer']:
num_from_optimizer += 1
if num_from_optimizer == len(remaining_tasks):
self.db_handler.set_tasks_to_redundant(remaining_tasks)
# [x] query parameters from designer
if task_set_completed and not designer_terminated:
# send new observations to designer, i.e. give designer the chance to update
observations = self.db_handler.get_circuit_evaluations(task_set_id)
if task_set.settings['use_library']:
observations.extend(prior_observations)
self.circuit_designer.provide_observations(task_set, observations)
# get new tasks from designer
new_tasks = self.circuit_designer.get_requested_tasks(task_set)
self.db_handler.add_tasks(new_tasks)
end = time.time()
reported_times.append(end - start)
reported_labels.append('processing special cases')
# give priority to tasks associated with currently running optimization iterations
sorted_tasks = remaining_tasks
print('# LOG | ... found %d remaining tasks ...' % len(sorted_tasks))
# [x] try to submit tasks to computing resources
# try to submit remaining tasks
start_0 = time.time()
num_available_resources = self.db_handler.get_num_available_resources(task_set)
if num_available_resources > 0:
submittable_tasks = sorted_tasks[:num_available_resources]
available_circuits = self.db_handler.get_validated_circuits()
if len(available_circuits) == 0:
start = time.time()
# no valid circuits available --> tell designer to generate more circuit parameters
print('# LOG | ... could not find validated circuits ...')
if not self.circuit_designer.is_busy(task_set):
# send observations to designer
observations = self.db_handler.get_circuit_evaluations(task_set.task_set_id)
if task_set.settings['use_library']:
observations.extend(prior_observations)
# tell designer to make more circuits
self.circuit_designer.design_new_circuits(task_set, observations = observations)
print('# LOG | ... called circuit designer ...')
reported_times.append(time.time() - start)
reported_labels.append('\tpinging_designer')
else:
# fetch valid circuits and submit to computing resources
num_submissions = np.minimum(len(submittable_tasks), len(available_circuits))
tasks = submittable_tasks[:num_submissions]
circuits = available_circuits[:num_submissions]
# attempt to submit circuits
start = time.time()
self.db_handler.reserve_circuits(circuits)
reported_times.append(time.time() - start)
reported_labels.append('\tsubmitting_task_0')
start = time.time()
submitted, not_submitted = self.circuit_submitter.submit(circuits, task_set)
reported_times.append(time.time() - start)
reported_labels.append('\tsubmitting_task_1')
# record successfully submitted circuits
start = time.time()
submitted_tasks = tasks
submitted_circuits = circuits
self.db_handler.set_tasks_to_submitted(submitted_tasks)
self.db_handler.link_submissions(submitted_tasks, submitted_circuits)
reported_times.append(time.time() - start)
reported_labels.append('\tsubmitting_task_3')
end_0 = time.time()
reported_times.append(end_0 - start_0)
reported_labels.append('processing remaining tasks')
# check if new circuits have been designed
start = time.time()
new_circuits = self.circuit_designer.get_circuits()
self.db_handler.add_new_circuits(new_circuits)
end = time.time()
reported_times.append(end - start)
reported_labels.append('getting circuits from designer')
# query validator for validated circuits
start = time.time()
validated_circuits = self.circuit_validator.get_validated_circuits()
self.db_handler.store_validated_circuits(validated_circuits)
end = time.time()
reported_times.append(end - start)
reported_labels.append('getting circuits from validator')
# submit new circuits to validator
start_0 = time.time()
new_circuits = self.db_handler.get_new_circuits()
end_0 = time.time()
reported_times.append(end_0 - start_0)
reported_labels.append('\tsubmitting circuits to validator 0')
start_1 = time.time()
print('# LOG | ... processing %d new circuits ...' % len(new_circuits))
self.circuit_validator.validate_circuits(new_circuits)
end_1 = time.time()
reported_times.append(end_1 - start_1)
reported_labels.append('submitting circuits to validator 1')
# collect criticized circuits
start = time.time()
criticized_circuit_results = self.circuit_critic.get_criticized_circuits()
criticized_circuits = [result[0] for result in criticized_circuit_results]
id_dicts = [result[1] for result in criticized_circuit_results]
print('# LOG | ... found %d criticized circuits ...' % len(criticized_circuits))
self.db_handler.store_criticized_circuits(criticized_circuits, id_dicts)
end = time.time()
reported_times.append(end - start)
reported_labels.append('collecting circuits from critic')
# check if critic requires new tasks
start = time.time()
new_circuits = self.circuit_critic.get_requested_tasks()
for circuit in new_circuits:
self.circuit_submitter.submit(circuit, task_set)
self.db_handler.report_circuit_submission()
end = time.time()
reported_times.append(end - start)
reported_labels.append('submitting tasks from critic')
# report progress
start = time.time()
progress_info = self.db_handler.get_task_set_progress_info(task_set, time.time() - start_time)
print('PROGRESS:\n%s (%.3f)' % (progress_info, time.time() - total_start))
end = time.time()
reported_times.append(end - start)
reported_labels.append('getting progress report')
iteration += 1
# update abortion criteria
start = time.time()
task_set_completed = self.db_handler.task_set_completed(task_set_id)
designer_terminated = self.circuit_designer.designer_terminated(task_set)
end = time.time()
reported_times.append(end - start)
reported_labels.append('updating abortion criteria')
# check if the designer requests any new tasks
start = time.time()
new_tasks = self.circuit_designer.get_requested_tasks(task_set)
self.db_handler.add_tasks(new_tasks)
end = time.time()
reported_times.append(end - start)
reported_labels.append('getting tasks from designer')
# query submitter for received spectra
start = time.time()
merit_evaluation_circuits, newly_computed_circuits = [], []
computed_circuits = self.circuit_submitter.get_computed_circuits()
for circuit in computed_circuits:
if 'merit_re-eval' in circuit:
merit_evaluation_circuits.append(circuit)
self.db_handler.report_circuit_computation()
else:
newly_computed_circuits.append(circuit)
reported_times.append(time.time() - start)
reported_labels.append('\tprocessing merit evaluations 0')
unsuccessful = True
id_dicts = self.db_handler.set_tasks_to_computed(newly_computed_circuits)
while len(id_dicts) < len(newly_computed_circuits):
id_dicts = self.db_handler.set_tasks_to_computed(newly_computed_circuits)
self.circuit_critic.criticize_circuits(newly_computed_circuits, task_set, id_dicts)
self.circuit_critic.report_reevaluations(merit_evaluation_circuits)
self.db_handler.print_pending_updates(time.time() - start_time)
self.db_handler.synchronize()
toc = time.time()
new_line = '@@@ Timing: %.5f @@@ | NUM_THREADS: %d\n' % ((toc - tic), threading.active_count())
print(new_line)
content = open('log_threads', 'a')
content.write('%.3f\t%d\n' % (time.time() - start_time, threading.active_count()))
content.close()
time.sleep(.2) # <== THIS SLEEP IS ESSENTIAL; DO NOT REMOVE
self.db_handler.set_circuits_to_unused()
def _run_filtering(self, task_set):
while self.db_handler.is_updating():
time.sleep(0.05)
self.db_handler.filter_for_best_performing(task_set.settings['designer_options'])
# wait for completion of update_requests
while self.db_handler.is_updating():
time.sleep(0.05)
def execute(self):
for task_set in self.task_sets:
print('# LOG | ... starting task "%s" ...' % task_set.task_set_name)
if task_set.task_type == 'calculation':
self._run_calculation(task_set)
elif task_set.task_type == 'filtering':
self._run_filtering(task_set)
elif task_set.task_type == 'db_query':
self._run_db_query(task_set)
print('# LOG | ... COMPLETED task "%s" ...' % task_set.task_set_name)
time.sleep(1)
def query(self, kind = None, **kwargs):
if kind == 'get_circuits_from_task':
return self.db_handler.get_circuits_from_task(kwargs['task'])
elif kind == 'get_trajectories':
return self.db_handler.get_trajectories(kwargs['task'])
elif kind == 'list_computing_tasks':
task_set_dicts = self.db_handler.list_computing_tasks()
return task_set_dicts
else:
raise NotImplementedError