From 1f932d267cea7d0bb9909bf000374019beef1542 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Thu, 4 Dec 2025 12:50:55 +1300 Subject: [PATCH] Report when each phase starts --- machine/jobs/build_clearml_helper.py | 9 +++++++++ machine/utils/phased_progress_reporter.py | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/machine/jobs/build_clearml_helper.py b/machine/jobs/build_clearml_helper.py index 029bb8bc..c85daa5b 100644 --- a/machine/jobs/build_clearml_helper.py +++ b/machine/jobs/build_clearml_helper.py @@ -91,6 +91,15 @@ def report_clearml_progress( "value": progress_status.step_count, } ) + if progress_status.phase_started is not None: + props.append( + { + "type": datetime, + "name": f"{progress_status.phase_stage}_started", + "description": "Phase Started", + "value": progress_status.phase_started, + } + ) if len(props) > 0: task.set_user_properties(*props) diff --git a/machine/utils/phased_progress_reporter.py b/machine/utils/phased_progress_reporter.py index a20553e0..7fc3706d 100644 --- a/machine/utils/phased_progress_reporter.py +++ b/machine/utils/phased_progress_reporter.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime from types import TracebackType from typing import Callable, ContextManager, Iterable, Optional, Sequence, Type @@ -19,6 +20,7 @@ class Phase: class PhaseProgressStatus(ProgressStatus): phase_stage: Optional[str] = None phase_step: Optional[int] = None + phase_started: Optional[datetime] = None class PhaseProgress(ContextManager[Callable[[ProgressStatus], None]]): @@ -70,6 +72,7 @@ def __init__(self, progress: Optional[Callable[[ProgressStatus], None]], phases: self._percent_completed = 0.0 self._step = 0 self._prev_phase_last_step = 0 + self._phase_started = None @property def phases(self) -> Sequence[Phase]: @@ -80,6 +83,7 @@ def current_phase(self) -> Optional[Phase]: return None if self._current_phase_index == -1 else self._phases[self._current_phase_index] def start_next_phase(self) -> PhaseProgress: + self._phase_started = datetime.now() self._prev_phase_last_step = self._step self._percent_completed += self._current_phase_percentage self._current_phase_index += 1 @@ -102,6 +106,7 @@ def _report(self, value: ProgressStatus) -> None: value.step_count, self.current_phase.stage if self.current_phase is not None else None, value.step, + self._phase_started, ) )