Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,11 +95,14 @@ jobs:
runs-on: ubuntu-latest
needs: changes
# Always run on push / schedule / manual dispatch; on PRs only when the
# paths probe matched or the PR is opted in via the `e2e` label.
# paths probe matched or the PR is opted in via the `e2e` label. Fail
# CLOSED: if the paths probe itself errors, run the suites rather than
# silently dropping the pre-merge gate (its outputs would read empty).
if: >-
!cancelled() &&
(github.event_name != 'pull_request' ||
contains(github.event.pull_request.labels.*.name, 'e2e') ||
needs.changes.result == 'failure' ||
needs.changes.outputs.e2e == 'true')
steps:
- uses: actions/checkout@v7
Expand Down Expand Up @@ -150,6 +153,7 @@ jobs:
!cancelled() &&
(github.event_name != 'pull_request' ||
contains(github.event.pull_request.labels.*.name, 'e2e') ||
needs.changes.result == 'failure' ||
needs.changes.outputs.e2e == 'true')
steps:
- uses: actions/checkout@v7
Expand Down
12 changes: 12 additions & 0 deletions internal/cli/doctor.go
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,18 @@ func summarizeDoctor(results []doctor.Result, tok tokenState) (connected, ready
ready = healthLine{doctor.StatusFail,
"Not ready — dataset storage isn't available.",
fmt.Sprintf("Email support@tracebloc.io with the output of `%s doctor --diagnose`.", launcher())}
case by["Node capacity"].Status == doctor.StatusWarn &&
(strings.HasPrefix(by["Node capacity"].Detail, "couldn't read RESOURCE_REQUESTS") ||
strings.HasPrefix(by["Node capacity"].Detail, "could not list nodes")):
// checkNodeFit's Warn covers two different situations: a can't-check
// (RESOURCE_REQUESTS unreadable, nodes unlistable) and the soft GPU
// fallback below. For a can't-check we simply don't know whether a node
// can fit a training job, so report an honest can't-check — mirroring
// the Pod-health list-failure case above — never a ✔ that skipped the
// capacity probe (Bugbot). The GPU-soft Warn intentionally stays Ready:
// training still runs via the jobs-manager's CPU fallback.
ready = healthLine{doctor.StatusUnknown,
"Ready to run training — couldn't check free compute (run with --verbose)", ""}
case by["Node capacity"].Status == doctor.StatusFail:
ready = healthLine{doctor.StatusFail,
"Not ready — not enough free compute to start a training.",
Expand Down
35 changes: 35 additions & 0 deletions internal/cli/doctor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,15 @@ func TestSummarizeDoctor(t *testing.T) {
}
return out
}
withDetail := func(base []doctor.Result, name string, s doctor.Status, detail string) []doctor.Result {
out := with(base, name, s)
for i := range out {
if out[i].Name == name {
out[i].Detail = detail
}
}
return out
}

t.Run("all healthy → both OK", func(t *testing.T) {
c, r := summarizeDoctor(allOK, tokenOK)
Expand All @@ -260,6 +269,32 @@ func TestSummarizeDoctor(t *testing.T) {
}
})

t.Run("node capacity can't-check → ready can't-check, never green", func(t *testing.T) {
// Both can't-check Warn details (nodes unlistable, RESOURCE_REQUESTS
// unreadable) must roll up to an honest Unknown — a ✔ here would assert
// readiness without a capacity probe (Bugbot).
for _, detail := range []string{
"could not list nodes: nodes is forbidden",
"couldn't read RESOURCE_REQUESTS from jobs-manager — skipping node-fit",
} {
_, r := summarizeDoctor(withDetail(allOK, "Node capacity", doctor.StatusWarn, detail), tokenOK)
if r.status != doctor.StatusUnknown {
t.Errorf("%q: ready should be Unknown, got %v", detail, r.status)
}
if !strings.Contains(r.text, "couldn't check free compute") {
t.Errorf("%q: ready text should say couldn't check free compute, got %q", detail, r.text)
}
}
})

t.Run("node capacity GPU-soft warn → still ready", func(t *testing.T) {
_, r := summarizeDoctor(withDetail(allOK, "Node capacity", doctor.StatusWarn,
"no single Ready node satisfies cpu+memory AND nvidia.com/gpu — GPU jobs rely on the CPU fallback (needs cpu=2, memory=8Gi)"), tokenOK)
if r.status != doctor.StatusOK {
t.Errorf("GPU-soft warn must stay Ready (CPU fallback), got %v", r.status)
}
})

t.Run("token unreachable → connected Fail", func(t *testing.T) {
c, _ := summarizeDoctor(allOK, tokenUnreachable)
if c.status != doctor.StatusFail || !strings.Contains(c.text, "can't reach tracebloc") {
Expand Down
15 changes: 12 additions & 3 deletions internal/push/list_detailed.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,10 +236,19 @@ func applyTaskLabels(ctx context.Context, exec Executor, namespace, pod, contain
if !hasTaskColumn {
return
}
// The LATEST run's task per table — never MAX(task), which is a
// lexicographic string max and can pin a re-ingested (--overwrite,
// different --task) dataset to its OLD task. started_at ships in the same
// DDL that creates the journal, so it exists whenever the table does. Two
// runs in the same second tie; either answer is fine (same pathological
// re-ingest).
out, err := runMySQLQuery(ctx, exec, namespace, pod, container, fmt.Sprintf(
"SELECT table_name, COALESCE(MAX(task),'') FROM `%s`.`%s` "+
"WHERE task IS NOT NULL GROUP BY table_name",
IngestionDatabase, ingestRunsTable))
"SELECT r.table_name, COALESCE(r.task,'') FROM `%s`.`%s` r "+
"JOIN (SELECT table_name, MAX(started_at) ms FROM `%s`.`%s` "+
"WHERE task IS NOT NULL GROUP BY table_name) m "+
"ON r.table_name = m.table_name AND r.started_at = m.ms "+
"WHERE r.task IS NOT NULL",
IngestionDatabase, ingestRunsTable, IngestionDatabase, ingestRunsTable))
if err != nil {
return // best-effort: the listing still renders with inferred modality
}
Expand Down
Loading