From cde6863cf8d3120a1d69c931dfcf47e5e920dd03 Mon Sep 17 00:00:00 2001 From: Paul Hinze Date: Wed, 13 May 2026 18:53:43 -0500 Subject: [PATCH] deploy: stable upload progress with honest ETA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Uploading artifacts: N% — uploaded / ~TOTAL at SPEED" line had a total estimate that wandered all over the place during slow uploads (observed: ~974 KB initial, climbing to ~80 MB, periodically dropping back to ~20 MB). The math was sampling uncompressed-bytes-in and compressed-bytes-out on different sides of a gzip buffer plus io.Pipe, so under network back-pressure their ratio was meaningless. Drop the synthetic total entirely. The percentage still works because both numerator (written) and denominator (totalUncompressed) only count the delta-filtered subset of files. Add an ETA extrapolated in the time domain: elapsed * (1 - frac) / frac. This avoids the unit-mixing that broke the old projection. A 5s warmup gates it so the first few wildly-fluctuating ticks do not show, and the display uses "(eta ~Xh Ym)" with a tilde to telegraph that it is approximate. Closes MIR-1136 --- cli/commands/deploy.go | 27 +++++++++--- cli/commands/deploy_test.go | 79 +++++++++++++++++++++++++++++++++++ cli/commands/deploy_ui.go | 14 ++++--- pkg/progress/upload/upload.go | 10 ++--- 4 files changed, 114 insertions(+), 16 deletions(-) create mode 100644 cli/commands/deploy_test.go diff --git a/cli/commands/deploy.go b/cli/commands/deploy.go index b21b57d9..9c2fb5d3 100644 --- a/cli/commands/deploy.go +++ b/cli/commands/deploy.go @@ -640,11 +640,14 @@ func Deploy(ctx *Context, opts struct { if progress.Fraction > 0 && time.Since(lastPrintTime) >= 500*time.Millisecond { lastPrintTime = time.Now() fmt.Fprintf(os.Stderr, "\r\033[K") // Clear to end of line - fmt.Fprintf(os.Stderr, "Uploading artifacts: %d%% — %s / ~%s at %s", + line := fmt.Sprintf("Uploading artifacts: %d%% — %s at %s", int(progress.Fraction*100), upload.FormatBytes(progress.BytesRead), - upload.FormatBytes(progress.EstimatedTotalBytes), upload.FormatSpeed(progress.BytesPerSecond)) + if progress.ETA > 0 { + line += fmt.Sprintf(" (eta ~%s)", upload.FormatDuration(progress.ETA)) + } + fmt.Fprint(os.Stderr, line) } }) r = progressReader @@ -927,15 +930,27 @@ func Deploy(ctx *Context, opts struct { return nil } -// enrichUploadProgress fills in Fraction and EstimatedTotalBytes on a Progress -// snapshot using the atomic uncompressed-byte counter and the known total. +// enrichUploadProgress fills in Fraction and ETA on a Progress snapshot using +// the atomic uncompressed-byte counter and the known total uncompressed size. +// +// Fraction is computed against uncompressed source bytes (not compressed bytes +// sent over the wire). We deliberately avoid projecting a compressed total: +// the source-bytes counter and the network-bytes counter are sampled on +// different sides of a gzip buffer plus io.Pipe, so their ratio swings wildly +// under back-pressure and produces a misleading "estimated total." +// +// ETA is extrapolated in the time domain — elapsed * (1 - frac) / frac — which +// avoids the unit-mixing that broke the old compressed-total math. It's only +// emitted after a brief warmup so the first few ticks (when Fraction is near +// zero) don't produce nonsense. func enrichUploadProgress(p *upload.Progress, written *atomic.Int64, totalUncompressed int64) { if totalUncompressed <= 0 { return } p.Fraction = float64(written.Load()) / float64(totalUncompressed) - if p.Fraction > 0 { - p.EstimatedTotalBytes = int64(float64(p.BytesRead) / p.Fraction) + if p.Fraction > 0 && p.Fraction < 1 && p.Duration >= 5*time.Second { + remaining := float64(p.Duration) * (1 - p.Fraction) / p.Fraction + p.ETA = time.Duration(remaining) } } diff --git a/cli/commands/deploy_test.go b/cli/commands/deploy_test.go new file mode 100644 index 00000000..7b8108cd --- /dev/null +++ b/cli/commands/deploy_test.go @@ -0,0 +1,79 @@ +package commands + +import ( + "sync/atomic" + "testing" + "time" + + "miren.dev/runtime/pkg/progress/upload" +) + +func TestEnrichUploadProgress(t *testing.T) { + const total int64 = 1_000_000 + + cases := []struct { + name string + written int64 + bytesRead int64 + elapsed time.Duration + wantFraction float64 + wantETA time.Duration + }{ + { + name: "zero total skips entirely", + written: 100, + bytesRead: 50, + elapsed: 5 * time.Second, + wantFraction: 0, + wantETA: 0, + }, + { + name: "warmup period suppresses ETA", + written: 10_000, + bytesRead: 5_000, + elapsed: 4 * time.Second, + wantFraction: 0.01, + wantETA: 0, + }, + { + name: "steady-state ETA extrapolates from elapsed", + written: 100_000, + bytesRead: 50_000, + elapsed: 10 * time.Second, + wantFraction: 0.1, + wantETA: 90 * time.Second, + }, + { + name: "completed upload reports no ETA", + written: total, + bytesRead: 500_000, + elapsed: 60 * time.Second, + wantFraction: 1.0, + wantETA: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var written atomic.Int64 + written.Store(tc.written) + + p := upload.Progress{ + BytesRead: tc.bytesRead, + Duration: tc.elapsed, + } + totalArg := total + if tc.name == "zero total skips entirely" { + totalArg = 0 + } + enrichUploadProgress(&p, &written, int64(totalArg)) + + if p.Fraction != tc.wantFraction { + t.Errorf("Fraction = %v, want %v", p.Fraction, tc.wantFraction) + } + if p.ETA != tc.wantETA { + t.Errorf("ETA = %v, want %v", p.ETA, tc.wantETA) + } + }) + } +} diff --git a/cli/commands/deploy_ui.go b/cli/commands/deploy_ui.go index e2b7c35e..06c63eaf 100644 --- a/cli/commands/deploy_ui.go +++ b/cli/commands/deploy_ui.go @@ -112,8 +112,8 @@ type deployInfo struct { finalUploadSpeed float64 // Upload progress estimation - uploadPct float64 // 0.0–1.0 fraction, -1 if unknown - uploadEstTotal int64 // estimated compressed total bytes + uploadPct float64 // 0.0–1.0 fraction, -1 if unknown + uploadETA time.Duration // estimated time remaining; 0 if unknown // Source cache info cachedFiles int32 @@ -275,7 +275,7 @@ func (m *deployInfo) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.uploadBytes = msg.BytesRead m.finalUploadSpeed = msg.BytesPerSecond m.uploadPct = msg.Fraction - m.uploadEstTotal = msg.EstimatedTotalBytes + m.uploadETA = msg.ETA // Continue reading upload progress if m.uploadProgress != nil { @@ -354,10 +354,14 @@ func (m *deployInfo) View() string { if pct < 0 { pct = 0 } - pctStr := deployPrefixStyle.Render(fmt.Sprintf("%d%% — %s at %s", + pctText := fmt.Sprintf("%d%% — %s at %s", int(pct*100), upload.FormatBytes(m.uploadBytes), - m.uploadSpeed)) + m.uploadSpeed) + if m.uploadETA > 0 { + pctText += fmt.Sprintf(" (eta ~%s)", upload.FormatDuration(m.uploadETA)) + } + pctStr := deployPrefixStyle.Render(pctText) currentLine = fmt.Sprintf(" %s Uploading artifacts...\n %s %s", m.uploadSpin.View(), m.prog.ViewAs(pct), pctStr) } else if m.currentPhase != "completed" { diff --git a/pkg/progress/upload/upload.go b/pkg/progress/upload/upload.go index 2c43a99c..2f4b691e 100644 --- a/pkg/progress/upload/upload.go +++ b/pkg/progress/upload/upload.go @@ -19,11 +19,11 @@ type ProgressReader struct { // Progress represents a snapshot of the current upload stats. type Progress struct { - BytesRead int64 - BytesPerSecond float64 - Duration time.Duration - Fraction float64 // 0.0–1.0 upload fraction complete - EstimatedTotalBytes int64 // projected total compressed bytes; 0 if unknown + BytesRead int64 + BytesPerSecond float64 + Duration time.Duration + Fraction float64 // 0.0–1.0 upload fraction complete + ETA time.Duration // estimated time remaining; 0 if unknown } // NewProgressReader creates a new progress tracking reader that wraps the given io.Reader.