Report when each phase starts

machine/jobs/build_clearml_helper.py

@@ -91,6 +91,15 @@ def report_clearml_progress(
                         "value": progress_status.step_count,
                     }
                 )
+            if progress_status.phase_started is not None:
+                props.append(
+                    {
+                        "type": datetime,
+                        "name": f"{progress_status.phase_stage}_started",
+                        "description": "Phase Started",
+                        "value": progress_status.phase_started,
+                    }
+                )
     if len(props) > 0:
         task.set_user_properties(*props)
 

machine/utils/phased_progress_reporter.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from datetime import datetime
 from types import TracebackType
 from typing import Callable, ContextManager, Iterable, Optional, Sequence, Type
 
@@ -19,6 +20,7 @@ class Phase:
 class PhaseProgressStatus(ProgressStatus):
     phase_stage: Optional[str] = None
     phase_step: Optional[int] = None
+    phase_started: Optional[datetime] = None
 
 
 class PhaseProgress(ContextManager[Callable[[ProgressStatus], None]]):
@@ -70,6 +72,7 @@ def __init__(self, progress: Optional[Callable[[ProgressStatus], None]], phases:
         self._percent_completed = 0.0
         self._step = 0
         self._prev_phase_last_step = 0
+        self._phase_started = None
 
     @property
     def phases(self) -> Sequence[Phase]:
@@ -80,6 +83,7 @@ def current_phase(self) -> Optional[Phase]:
         return None if self._current_phase_index == -1 else self._phases[self._current_phase_index]
 
     def start_next_phase(self) -> PhaseProgress:
+        self._phase_started = datetime.now()
         self._prev_phase_last_step = self._step
         self._percent_completed += self._current_phase_percentage
         self._current_phase_index += 1
@@ -102,6 +106,7 @@ def _report(self, value: ProgressStatus) -> None:
                 value.step_count,
                 self.current_phase.stage if self.current_phase is not None else None,
                 value.step,
+                self._phase_started,
             )
         )