From 69c6855786536d9869fec67c2296e70b343f29d1 Mon Sep 17 00:00:00 2001 From: Mark Muetzelfeldt Date: Wed, 1 Nov 2023 20:26:50 +0000 Subject: [PATCH] Minor changes to timing and fix logic bug in task assignment. --- remake/global_timer.py | 14 ++++++++------ remake/remake_cmd.py | 1 + remake/task_control.py | 16 +++++++++------- remake/task_rule.py | 18 +++++++----------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/remake/global_timer.py b/remake/global_timer.py index b64a618..314f17a 100644 --- a/remake/global_timer.py +++ b/remake/global_timer.py @@ -31,12 +31,14 @@ def __str__(self): for k in self.timers.keys(): k1, k2 = k timers = self.timers[k] - times_ms = [t.microseconds for t in timers] - time_total_ms = np.sum(times_ms) - time_mean_ms = np.mean(times_ms) - time_std_ms = np.std(times_ms) - count = len(times_ms) - output.append((f'{k1} -> {k2}', f'{time_total_ms / 1e6:.2g}s', f'{time_mean_ms / 1e6:.2g}s', f'(+/- {time_std_ms / 1e6:.2g}s)', f'{count}')) + # print(timers) + times = [t.total_seconds() for t in timers] + # print(times) + time_total = np.sum(times) + time_mean = np.mean(times) + time_std = np.std(times) + count = len(times) + output.append((f'{k1} -> {k2}', f'{time_total:.2g}s', f'{time_mean:.2g}s', f'(+/- {time_std:.2g}s)', f'{count}')) return f'{self.name}\n' + '=' * len(self.name) + '\n' + tabulate(output, headers=('tx', 'total', 'mean', 'std', 'count')) def start(self): diff --git a/remake/remake_cmd.py b/remake/remake_cmd.py index c14f3f8..568a6dd 100644 --- a/remake/remake_cmd.py +++ b/remake/remake_cmd.py @@ -360,6 +360,7 @@ def remake_cmd(argv: Union[List[str], None] = None) -> None: if args.subcmd_name != 'monitor': setup_stdout_logging(loglevel, colour=colour) + logger.warn('using experimental version with no track_files') parser.dispatch() diff --git a/remake/task_control.py b/remake/task_control.py index 7336095..5dc137f 100644 --- a/remake/task_control.py +++ b/remake/task_control.py @@ -399,8 +399,12 @@ def _assign_tasks(self): # completed: task has been run and does not need to be rerun. # pending: task has been run and needs to be rerun. # remaining: task either needs to be rerun, or has previous tasks that need to be rerun. - # import ipdb; ipdb.set_trace() + def trigger(task): + return False + for task in self.sorted_tasks.keys(): + if trigger(task): + import ipdb; ipdb.set_trace() logger.debug(f' assign task: {task}') requires_rerun = self.task_requires_rerun(task) @@ -419,7 +423,7 @@ def _assign_tasks(self): # if status != 'remaining' and requires_rerun & self.remake_on: # status = 'pending' else: - # status = 'remaining' + status = 'pending' # TODO: out of date. # Reasons task can be cannot run: # 1: one of its prev tasks cannot be run. @@ -433,11 +437,9 @@ def _assign_tasks(self): prev_task in self.remaining_tasks or isinstance(prev_task, RescanFileTask)): status = 'remaining' - if prev_task in self.cannot_run_tasks: - status = 'cannot_run' - break - else: - status = 'pending' + # if prev_task in self.cannot_run_tasks: + # status = 'cannot_run' + # break # for input_path in task.inputs.values(): # if input_path not in self.output_task_map and not input_path.exists(): # status = 'cannot_run' diff --git a/remake/task_rule.py b/remake/task_rule.py index e736a7f..414aaad 100644 --- a/remake/task_rule.py +++ b/remake/task_rule.py @@ -64,7 +64,8 @@ def __new__(mcs, clsname, bases, attrs): var_matrix = attrs.get('var_matrix', None) depends_on.extend(attrs.get('depends_on', [])) logger.debug(f' depends on: {depends_on}') - loop_timer = get_global_timer(str(clsname)) + + loop_timer = get_global_timer(str(clsname) + '_loop_var_timer') create_inputs_fn = RemakeMetaclass._get_create_inputs_ouputs_fn(attrs['rule_inputs']) create_outputs_fn = RemakeMetaclass._get_create_inputs_ouputs_fn(attrs['rule_outputs']) @@ -102,25 +103,20 @@ def __new__(mcs, clsname, bases, attrs): newcls.tasks.append(task) remake.task_ctrl.add(task) loop_timer(5) - print(loop_timer) + + logger.debug('\n' + str(loop_timer)) task_ctrl_add_timer = get_global_timer('task_ctrl_add') - print(task_ctrl_add_timer) + logger.debug('\n' + str(task_ctrl_add_timer)) task_ctrl_add_timer.reset() create_task_metadata_timer = get_global_timer('create_task_metadata_timer') - print(create_task_metadata_timer) + logger.debug('\n' + str(create_task_metadata_timer)) create_task_metadata_timer.reset() PathMetadata_timer = get_global_timer('PathMetadata_timer') - print(PathMetadata_timer) + logger.debug('\n' + str(PathMetadata_timer)) PathMetadata_timer.reset() - - # task_init_timer = get_global_timer(str(newcls) + '__init__') - # print(task_init_timer) - # cond_input_timer = get_global_timer('cond_input_timer') - # print(cond_input_timer) - # cond_input_timer.reset() else: logger.debug(f' creating instance of {clsname}') inputs = create_inputs_fn(**{})