diff --git a/src/together/cli/api/finetune.py b/src/together/cli/api/finetune.py index dbd2eed..a1377c8 100644 --- a/src/together/cli/api/finetune.py +++ b/src/together/cli/api/finetune.py @@ -9,10 +9,11 @@ import click from click.core import ParameterSource # type: ignore[attr-defined] from rich import print as rprint +from rich.json import JSON from tabulate import tabulate from together import Together -from together.cli.api.utils import BOOL_WITH_AUTO, INT_WITH_MAX +from together.cli.api.utils import BOOL_WITH_AUTO, INT_WITH_MAX, generate_progress_bar from together.types.finetune import ( DownloadCheckpointType, FinetuneEventType, @@ -435,6 +436,9 @@ def list(ctx: click.Context) -> None: "Price": f"""${ finetune_price_to_dollars(float(str(i.total_price))) }""", # convert to string for mypy typing + "Progress": generate_progress_bar( + i, datetime.now().astimezone(), use_rich=False + ), } ) table = tabulate(display_list, headers="keys", tablefmt="grid", showindex=True) @@ -454,7 +458,15 @@ def retrieve(ctx: click.Context, fine_tune_id: str) -> None: # remove events from response for cleaner output response.events = None - click.echo(json.dumps(response.model_dump(exclude_none=True), indent=4)) + rprint(JSON.from_data(response.model_dump(exclude_none=True))) + progress_text = generate_progress_bar( + response, datetime.now().astimezone(), use_rich=True + ) + status = "Unknown" + if response.status is not None: + status = response.status.value + prefix = f"Status: [bold]{status}[/bold]," + rprint(f"{prefix} {progress_text}") @fine_tuning.command() diff --git a/src/together/cli/api/utils.py b/src/together/cli/api/utils.py index 08dfe49..3bd2844 100644 --- a/src/together/cli/api/utils.py +++ b/src/together/cli/api/utils.py @@ -1,10 +1,17 @@ from __future__ import annotations +import math +import re from gettext import gettext as _ from typing import Literal +from datetime import datetime import click +from together.types.finetune import FinetuneResponse, COMPLETED_STATUSES + +_PROGRESS_BAR_WIDTH = 40 + class AutoIntParamType(click.ParamType): name = "integer_or_max" @@ -49,3 +56,84 @@ def convert( INT_WITH_MAX = AutoIntParamType() BOOL_WITH_AUTO = BooleanWithAutoParamType() + + +def _human_readable_time(timedelta: float) -> str: + """Convert a timedelta to a compact human-readble string + Examples: + 00:00:10 -> 10s + 01:23:45 -> 1h 23min 45s + 1 Month 23 days 04:56:07 -> 1month 23d 4h 56min 7s + Args: + timedelta (float): The timedelta in seconds to convert. + Returns: + A string representing the timedelta in a human-readable format. + """ + units = [ + (30 * 24 * 60 * 60, "month"), # 30 days + (24 * 60 * 60, "d"), + (60 * 60, "h"), + (60, "min"), + (1, "s"), + ] + + total_seconds = int(timedelta) + parts = [] + + for unit_seconds, unit_name in units: + if total_seconds >= unit_seconds: + value = total_seconds // unit_seconds + total_seconds %= unit_seconds + parts.append(f"{value}{unit_name}") + + return " ".join(parts) if parts else "0s" + + +def generate_progress_bar( + finetune_job: FinetuneResponse, current_time: datetime, use_rich: bool = False +) -> str: + """Generate a progress bar for a finetune job. + Args: + finetune_job: The finetune job to generate a progress bar for. + current_time: The current time. + use_rich: Whether to use rich formatting. + Returns: + A string representing the progress bar. + """ + progress = "Progress: [bold red]unavailable[/bold red]" + if finetune_job.status in COMPLETED_STATUSES: + progress = "Progress: [bold green]completed[/bold green]" + elif finetune_job.updated_at is not None: + # Replace 'Z' with '+00:00' for Python 3.10 compatibility + updated_at_str = finetune_job.updated_at.replace("Z", "+00:00") + update_at = datetime.fromisoformat(updated_at_str).astimezone() + + if finetune_job.progress is not None: + if current_time < update_at: + return progress + + if not finetune_job.progress.estimate_available: + return progress + + if finetune_job.progress.seconds_remaining <= 0: + return progress + + elapsed_time = (current_time - update_at).total_seconds() + ratio_filled = min( + elapsed_time / finetune_job.progress.seconds_remaining, 1.0 + ) + percentage = ratio_filled * 100 + filled = math.ceil(ratio_filled * _PROGRESS_BAR_WIDTH) + bar = "█" * filled + "░" * (_PROGRESS_BAR_WIDTH - filled) + time_left = "N/A" + if finetune_job.progress.seconds_remaining > elapsed_time: + time_left = _human_readable_time( + finetune_job.progress.seconds_remaining - elapsed_time + ) + time_text = f"{time_left} left" + progress = f"Progress: {bar} [bold]{percentage:>3.0f}%[/bold] [yellow]{time_text}[/yellow]" + + if use_rich: + return progress + + return re.sub(r"\[/?[^\]]+\]", "", progress) diff --git a/src/together/types/finetune.py b/src/together/types/finetune.py index 286932e..37250ae 100644 --- a/src/together/types/finetune.py +++ b/src/together/types/finetune.py @@ -28,6 +28,14 @@ class FinetuneJobStatus(str, Enum): STATUS_COMPLETED = "completed" +COMPLETED_STATUSES = [ + FinetuneJobStatus.STATUS_ERROR, + FinetuneJobStatus.STATUS_USER_ERROR, + FinetuneJobStatus.STATUS_COMPLETED, + FinetuneJobStatus.STATUS_CANCELLED, +] + + class FinetuneEventLevels(str, Enum): """ Fine-tune job event status levels @@ -167,6 +175,15 @@ class TrainingMethodDPO(TrainingMethod): simpo_gamma: float | None = None +class FinetuneProgress(BaseModel): + """ + Fine-tune job progress + """ + + estimate_available: bool = False + seconds_remaining: float = 0 + + class FinetuneRequest(BaseModel): """ Fine-tune request type @@ -297,6 +314,8 @@ class FinetuneResponse(BaseModel): train_on_inputs: StrictBool | Literal["auto"] | None = "auto" from_checkpoint: str | None = None + progress: FinetuneProgress | None = None + @field_validator("training_type") @classmethod def validate_training_type(cls, v: TrainingType) -> TrainingType: diff --git a/tests/unit/test_cli_utils.py b/tests/unit/test_cli_utils.py new file mode 100644 index 0000000..02e6182 --- /dev/null +++ b/tests/unit/test_cli_utils.py @@ -0,0 +1,407 @@ +from datetime import datetime, timezone + +import pytest + +from together.cli.api.utils import generate_progress_bar +from together.types.finetune import ( + FinetuneResponse, + FinetuneProgress, + FinetuneJobStatus, +) + + +def create_finetune_response( + status: FinetuneJobStatus = FinetuneJobStatus.STATUS_RUNNING, + updated_at: str = "2024-01-01T12:00:00Z", + progress: FinetuneProgress | None = None, + job_id: str = "ft-test-123", +) -> FinetuneResponse: + """Helper function to create FinetuneResponse objects for testing. + + Args: + status: The job status. + updated_at: The updated timestamp in ISO format. + progress: Optional FinetuneProgress object. + job_id: The fine-tune job ID. + + Returns: + A FinetuneResponse object for testing. + """ + return FinetuneResponse( + id=job_id, + progress=progress, + updated_at=updated_at, + status=status, + ) + + +class TestGenerateProgressBarGeneral: + """General test cases for normal operation.""" + + def test_progress_unavailable_when_none(self): + """Test that progress shows unavailable when progress field is None.""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + finetune_job = create_finetune_response(progress=None) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert result == "Progress: [bold red]unavailable[/bold red]" + + def test_progress_unavailable_when_not_set(self): + """Test that progress shows unavailable when field is not provided.""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + finetune_job = create_finetune_response() + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert result == "Progress: [bold red]unavailable[/bold red]" + + def test_progress_bar_at_start(self): + """Test progress bar display when job just started (low percentage).""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=1000.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + # 10 seconds elapsed / 1000 seconds remaining = 0.01 ratio = 1% progress + # 0.01 * 40 = 0.4, ceil(0.4) = 1 filled bar + assert ( + result + == "Progress: █░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ [bold] 1%[/bold] [yellow]16min 30s left[/yellow]" + ) + + def test_progress_bar_at_midpoint(self): + """Test progress bar at approximately 50% completion.""" + current_time = datetime(2024, 1, 1, 12, 1, 0, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + # 60 seconds elapsed / 60 seconds remaining = 1.0 ratio = 100% progress + # 1.0 * 40 = 40 filled bars + assert ( + result + == "Progress: ████████████████████████████████████████ [bold]100%[/bold] [yellow]N/A left[/yellow]" + ) + + def test_progress_bar_near_completion(self): + """Test progress bar when job is almost complete.""" + current_time = datetime(2024, 1, 1, 12, 5, 0, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=30.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + # 300 seconds elapsed / 30 seconds remaining = 10.0 ratio = 1000% progress + # 10.0 * 40 = 400, ceil(400) = 400, but width is 40 so all filled + assert ( + result + == "Progress: ████████████████████████████████████████ [bold]100%[/bold] [yellow]N/A left[/yellow]" + ) + + def test_progress_bar_contains_rich_formatting(self): + """Test that progress bar includes expected Rich markup formatting.""" + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + # 30 seconds elapsed / 60 seconds remaining = 0.5 ratio = 50% progress + # 0.5 * 40 = 20 filled bars + assert ( + result + == "Progress: ████████████████████░░░░░░░░░░░░░░░░░░░░ [bold] 50%[/bold] [yellow]30s left[/yellow]" + ) + + +class TestGenerateProgressBarRichFormatting: + """Test cases for use_rich parameter.""" + + def test_rich_formatting_removed_when_use_rich_false(self): + """Test that rich formatting tags are removed when use_rich=False.""" + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=False) + + assert ( + result == "Progress: ████████████████████░░░░░░░░░░░░░░░░░░░░ 50% 30s left" + ) + + def test_rich_formatting_preserved_when_use_rich_true(self): + """Test that rich formatting tags are preserved when use_rich=True.""" + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert ( + result + == "Progress: ████████████████████░░░░░░░░░░░░░░░░░░░░ [bold] 50%[/bold] [yellow]30s left[/yellow]" + ) + + def test_completed_status_formatting_removed(self): + """Test that completed status formatting is removed when use_rich=False.""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + status=FinetuneJobStatus.STATUS_COMPLETED, progress=None + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=False) + + assert result == "Progress: completed" + + def test_unavailable_status_formatting_removed(self): + """Test that unavailable status formatting is removed when use_rich=False.""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + finetune_job = create_finetune_response(progress=None) + + result = generate_progress_bar(finetune_job, current_time, use_rich=False) + + assert result == "Progress: unavailable" + + def test_rich_formatting_removed_at_completion(self): + """Test that rich formatting is removed at 100% when use_rich=False.""" + current_time = datetime(2024, 1, 1, 12, 1, 0, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=False) + + assert ( + result == "Progress: ████████████████████████████████████████ 100% N/A left" + ) + + def test_default_behavior_strips_formatting(self): + """Test that rich formatting is removed by default (use_rich not specified).""" + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time) + + assert ( + result == "Progress: ████████████████████░░░░░░░░░░░░░░░░░░░░ 50% 30s left" + ) + + def test_content_consistency_between_modes(self): + """Test that use_rich=True and use_rich=False have same content, just different formatting.""" + import re + + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result_with_rich = generate_progress_bar( + finetune_job, current_time, use_rich=True + ) + result_without_rich = generate_progress_bar( + finetune_job, current_time, use_rich=False + ) + + stripped_rich = re.sub(r"\[/?[^\]]+\]", "", result_with_rich) + assert stripped_rich == result_without_rich + + def test_all_rich_tag_types_removed(self): + """Test that all types of rich formatting tags are properly removed.""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + + # Test with completed status (has [bold green] tags) + completed_job = create_finetune_response( + status=FinetuneJobStatus.STATUS_COMPLETED, progress=None + ) + result_completed = generate_progress_bar( + completed_job, current_time, use_rich=False + ) + assert result_completed == "Progress: completed" + + # Test with unavailable status (has [bold red] tags) + unavailable_job = create_finetune_response(progress=None) + result_unavailable = generate_progress_bar( + unavailable_job, current_time, use_rich=False + ) + assert result_unavailable == "Progress: unavailable" + + @pytest.mark.parametrize( + "use_rich,expected_completed,expected_running", + [ + ( + True, + "Progress: [bold green]completed[/bold green]", + "Progress: ███████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ [bold] 17%[/bold] [yellow]50s left[/yellow]", + ), + ( + False, + "Progress: completed", + "Progress: ███████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 17% 50s left", + ), + ], + ) + def test_rich_parameter_with_different_statuses( + self, use_rich, expected_completed, expected_running + ): + """Test use_rich parameter works correctly with different job statuses.""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + + # Test completed status + completed_job = create_finetune_response( + status=FinetuneJobStatus.STATUS_COMPLETED, progress=None + ) + result = generate_progress_bar(completed_job, current_time, use_rich=use_rich) + assert result == expected_completed + + # Test running status + running_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + result = generate_progress_bar(running_job, current_time, use_rich=use_rich) + assert result == expected_running + + def test_progress_percentage_1_percent(self): + """Test progress bar at 1% completion.""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=1000.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=False) + assert ( + result + == "Progress: █░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 1% 16min 30s left" + ) + + def test_progress_percentage_75_percent(self): + """Test progress bar at 75% completion.""" + current_time = datetime(2024, 1, 1, 12, 0, 45, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=False) + assert ( + result == "Progress: ██████████████████████████████░░░░░░░░░░ 75% 15s left" + ) + + +class TestGenerateProgressBarCornerCases: + """Corner cases and edge conditions.""" + + def test_zero_seconds_remaining(self): + """Test handling of zero seconds remaining (potential division by zero).""" + current_time = datetime(2024, 1, 1, 12, 0, 10, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=0.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert result == "Progress: [bold red]unavailable[/bold red]" + + def test_very_small_remaining_time(self): + """Test with very small remaining time (< 1 second).""" + current_time = datetime(2024, 1, 1, 12, 0, 5, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=0.5) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert ( + result + == "Progress: ████████████████████████████████████████ [bold]100%[/bold] [yellow]N/A left[/yellow]" + ) + + def test_very_large_remaining_time(self): + """Test with very large remaining time (hours).""" + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress( + estimate_available=True, seconds_remaining=36000.0 + ) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert ( + result + == "Progress: █░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ [bold] 0%[/bold] [yellow]9h 59min 30s left[/yellow]" + ) + + def test_job_exceeding_estimate(self): + """Test when elapsed time exceeds original estimate (>100% progress).""" + current_time = datetime(2024, 1, 1, 14, 0, 0, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert ( + result + == "Progress: ████████████████████████████████████████ [bold]100%[/bold] [yellow]N/A left[/yellow]" + ) + + def test_timezone_aware_datetime(self): + """Test with different timezone for updated_at.""" + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + updated_at="2024-01-01T07:00:00-05:00", # Same as 12:00:00 UTC (EST = UTC-5) + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0), + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert ( + result + == "Progress: ████████████████████░░░░░░░░░░░░░░░░░░░░ [bold] 50%[/bold] [yellow]30s left[/yellow]" + ) + + def test_estimate_unavailable_flag(self): + """Test when estimate_available flag is False.""" + current_time = datetime(2024, 1, 1, 12, 0, 50, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=False, seconds_remaining=100.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert result == "Progress: [bold red]unavailable[/bold red]" + + def test_negative_elapsed_time_scenario(self): + """Test unusual case where current time appears before updated_at.""" + current_time = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + updated_at="2024-01-01T12:00:30Z", # In the "future" + progress=FinetuneProgress(estimate_available=True, seconds_remaining=100.0), + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=True) + + assert result == "Progress: [bold red]unavailable[/bold red]" + + def test_unicode_progress_bars_preserved(self): + """Test that unicode characters in progress bars are preserved after tag removal.""" + current_time = datetime(2024, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + finetune_job = create_finetune_response( + progress=FinetuneProgress(estimate_available=True, seconds_remaining=60.0) + ) + + result = generate_progress_bar(finetune_job, current_time, use_rich=False) + + assert ( + result == "Progress: ████████████████████░░░░░░░░░░░░░░░░░░░░ 50% 30s left" + )