From 4bbebcb6c01d5a2b659b153fdbac2f708c4d3b82 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint Date: Thu, 23 Jul 2026 17:16:23 +0200 Subject: [PATCH] Add pre-flight agent artifact validation to codefly ci run (#101) Resolve each affected service's pinned agent against its configured artifact sources before the phase loop runs, so an unpublished pin (a git tag with no downloadable release asset / OCI manifest) fails fast with a single legible report instead of an opaque mid-run 404. Co-Authored-By: Claude Opus 4.8 --- cmd/ci/agent_versions.go | 221 ++++++++++++++++++++++++++++++++++ cmd/ci/agent_versions_test.go | 147 ++++++++++++++++++++++ cmd/ci/run.go | 3 + 3 files changed, 371 insertions(+) create mode 100644 cmd/ci/agent_versions.go create mode 100644 cmd/ci/agent_versions_test.go diff --git a/cmd/ci/agent_versions.go b/cmd/ci/agent_versions.go new file mode 100644 index 00000000..4af042fd --- /dev/null +++ b/cmd/ci/agent_versions.go @@ -0,0 +1,221 @@ +package ci + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "path" + "strings" + "time" + + "github.com/codefly-dev/cli/pkg/cli" + "github.com/codefly-dev/core/agents/manager" + "github.com/codefly-dev/core/resources" +) + +// Seams for testing the artifact probe without reaching real registries. +var ( + resolveAgentLatest = manager.ResolveLatest + agentAlreadyLocal = manager.Downloaded + githubAssetURL = manager.DownloadURL + agentProbeClient = &http.Client{Timeout: 20 * time.Second} +) + +// agentSourceProbe is the outcome of resolving one agent against one source. +type agentSourceProbe struct { + label string + downloadable bool + detail string +} + +// agentArtifactStatus aggregates every source tried for a single agent pin. +type agentArtifactStatus struct { + agent *resources.Agent + sources []agentSourceProbe +} + +func (s agentArtifactStatus) downloadable() bool { + for _, source := range s.sources { + if source.downloadable { + return true + } + } + return false +} + +// validateAgentVersions resolves every affected service's pinned agent against +// the configured artifact sources before any CI phase spawns it. A pin that is +// tagged but has no downloadable artifact would otherwise surface as an opaque +// 404 deep inside a phase; here it fails fast with a single, legible report. +// +// Skipped entirely under CODEFLY_AGENT_SOURCE=local, where agents are resolved +// from local builds and no artifact is ever downloaded. +func validateAgentVersions(ctx context.Context, workspace *resources.Workspace, plan *Plan) error { + if manager.AgentSourceLocal() { + return nil + } + if plan == nil || len(plan.Services) == 0 { + return nil + } + cli.Header(2, "Pre-flight: validating agent versions") + + agents, err := collectPlanAgents(ctx, workspace, plan) + if err != nil { + return err + } + + var unpublished []agentArtifactStatus + for _, agent := range agents { + if _, err := resolveAgentLatest(ctx, agent); err != nil { + unpublished = append(unpublished, agentArtifactStatus{ + agent: agent, + sources: []agentSourceProbe{{label: "version resolution", detail: err.Error()}}, + }) + continue + } + if local, err := agentAlreadyLocal(ctx, agent); err == nil && local { + continue + } + status := probeAgentArtifact(ctx, agent) + if !status.downloadable() { + unpublished = append(unpublished, status) + } + } + + if len(unpublished) == 0 { + return nil + } + return errors.New(formatUnpublishedReport(unpublished)) +} + +// collectPlanAgents returns the distinct agents backing the affected services, +// keyed by kind+identifier so a shared pin is validated once. +func collectPlanAgents(ctx context.Context, workspace *resources.Workspace, plan *Plan) ([]*resources.Agent, error) { + seen := map[string]bool{} + var agents []*resources.Agent + for _, planned := range plan.Services { + ref, err := resources.ParseServiceWithOptionalModule(planned.Service) + if err != nil { + return nil, fmt.Errorf("parse affected service %q: %w", planned.Service, err) + } + module, err := workspace.LoadModuleFromName(ctx, ref.Module) + if err != nil { + return nil, fmt.Errorf("load module %q: %w", ref.Module, err) + } + service, err := module.LoadServiceFromName(ctx, ref.Name) + if err != nil { + return nil, fmt.Errorf("load service %q: %w", planned.Service, err) + } + if service.Agent == nil { + continue + } + agent := *service.Agent + if seen[agent.Unique()] { + continue + } + seen[agent.Unique()] = true + agents = append(agents, &agent) + } + return agents, nil +} + +// probeAgentArtifact HEADs the GitHub release asset and, when configured, the +// OCI manifest and Nix flake output for the agent. +func probeAgentArtifact(ctx context.Context, agent *resources.Agent) agentArtifactStatus { + status := agentArtifactStatus{agent: agent} + status.sources = append(status.sources, probeGitHubAsset(ctx, agent)) + status.sources = append(status.sources, probeOCIManifest(ctx, agent)) + if source, ok := probeNixFlake(ctx, agent); ok { + status.sources = append(status.sources, source) + } + return status +} + +func probeGitHubAsset(ctx context.Context, agent *resources.Agent) agentSourceProbe { + url := githubAssetURL(agent) + probe := agentSourceProbe{label: "GitHub release asset " + path.Base(url)} + req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + probe.detail = err.Error() + return probe + } + resp, err := agentProbeClient.Do(req) + if err != nil { + probe.detail = err.Error() + return probe + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + probe.downloadable = true + probe.detail = "ok" + return probe + } + probe.detail = fmt.Sprintf("%d", resp.StatusCode) + return probe +} + +func probeOCIManifest(ctx context.Context, agent *resources.Agent) agentSourceProbe { + registry := strings.TrimSpace(os.Getenv("AGENT_REGISTRY")) + if registry == "" { + return agentSourceProbe{label: "OCI registry", detail: "not configured (set AGENT_REGISTRY)"} + } + reference := fmt.Sprintf("%s/agents/%s/%s:%s", registry, agent.Publisher, agent.Name, agent.Version) + probe := agentSourceProbe{label: "OCI " + reference} + store := manager.NewOCIStoreFromEnv(slog.Default()) + if store == nil { + probe.detail = "not configured (set AGENT_REGISTRY)" + return probe + } + available, err := store.Available(ctx, agent) + if err != nil { + probe.detail = err.Error() + return probe + } + if available { + probe.downloadable = true + probe.detail = "ok" + return probe + } + probe.detail = "manifest not found" + return probe +} + +func probeNixFlake(ctx context.Context, agent *resources.Agent) (agentSourceProbe, bool) { + store := manager.NewNixStoreFromEnv(slog.Default()) + if store == nil { + return agentSourceProbe{}, false + } + probe := agentSourceProbe{label: "Nix flake " + os.Getenv("AGENT_NIX_FLAKE")} + available, err := store.Available(ctx, agent) + if err != nil { + probe.detail = err.Error() + return probe, true + } + if available { + probe.downloadable = true + probe.detail = "ok" + return probe, true + } + probe.detail = "flake output not found" + return probe, true +} + +func formatUnpublishedReport(statuses []agentArtifactStatus) string { + var b strings.Builder + plural := "pin" + if len(statuses) > 1 { + plural = "pins" + } + fmt.Fprintf(&b, "%d agent %s not downloadable in CI (no published artifact):\n", len(statuses), plural) + for _, status := range statuses { + fmt.Fprintf(&b, "\nagent %s is not published (no CI-downloadable artifact)\n", status.agent.Identifier()) + for _, source := range status.sources { + fmt.Fprintf(&b, " - %s: %s\n", source.label, source.detail) + } + b.WriteString(" -> tag + release the agent, or set AGENT_REGISTRY\n") + } + return strings.TrimRight(b.String(), "\n") +} diff --git a/cmd/ci/agent_versions_test.go b/cmd/ci/agent_versions_test.go new file mode 100644 index 00000000..ee4cf212 --- /dev/null +++ b/cmd/ci/agent_versions_test.go @@ -0,0 +1,147 @@ +package ci + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/codefly-dev/core/resources" +) + +func testAgent() *resources.Agent { + return &resources.Agent{ + Kind: resources.ServiceAgent, + Publisher: "codefly.dev", + Name: "redis", + Version: "0.0.74", + } +} + +func TestProbeGitHubAssetReportsMissingArtifact(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + restore := githubAssetURL + githubAssetURL = func(*resources.Agent) string { return server.URL + "/service-redis_0.0.74_linux_amd64.tar.gz" } + defer func() { githubAssetURL = restore }() + + probe := probeGitHubAsset(context.Background(), testAgent()) + if probe.downloadable { + t.Fatal("404 asset reported as downloadable") + } + if probe.detail != "404" { + t.Fatalf("detail = %q, want 404", probe.detail) + } + if !strings.Contains(probe.label, "service-redis_0.0.74_linux_amd64.tar.gz") { + t.Fatalf("label = %q, want the asset filename", probe.label) + } +} + +func TestProbeGitHubAssetReportsPublishedArtifact(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + restore := githubAssetURL + githubAssetURL = func(*resources.Agent) string { return server.URL + "/asset.tar.gz" } + defer func() { githubAssetURL = restore }() + + probe := probeGitHubAsset(context.Background(), testAgent()) + if !probe.downloadable { + t.Fatalf("200 asset reported as not downloadable: %q", probe.detail) + } +} + +func TestProbeOCIManifestNotConfigured(t *testing.T) { + t.Setenv("AGENT_REGISTRY", "") + probe := probeOCIManifest(context.Background(), testAgent()) + if probe.downloadable { + t.Fatal("unconfigured OCI reported as downloadable") + } + if !strings.Contains(probe.detail, "not configured") { + t.Fatalf("detail = %q, want 'not configured'", probe.detail) + } +} + +func TestProbeOCIManifestAvailable(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/manifests/0.0.74") { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + registry := strings.TrimPrefix(server.URL, "http://") + t.Setenv("AGENT_REGISTRY", registry) + + probe := probeOCIManifest(context.Background(), testAgent()) + if !probe.downloadable { + t.Fatalf("available OCI manifest reported as not downloadable: %q", probe.detail) + } + want := registry + "/agents/codefly.dev/redis:0.0.74" + if !strings.Contains(probe.label, want) { + t.Fatalf("label = %q, want reference %q", probe.label, want) + } +} + +func TestProbeAgentArtifactCombinesSources(t *testing.T) { + github := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer github.Close() + + restore := githubAssetURL + githubAssetURL = func(*resources.Agent) string { return github.URL + "/asset.tar.gz" } + defer func() { githubAssetURL = restore }() + t.Setenv("AGENT_REGISTRY", "") + + status := probeAgentArtifact(context.Background(), testAgent()) + if status.downloadable() { + t.Fatal("agent with no reachable source reported as downloadable") + } + if len(status.sources) != 2 { + t.Fatalf("sources = %d, want GitHub + OCI when Nix is unconfigured", len(status.sources)) + } +} + +func TestFormatUnpublishedReportListsEverySource(t *testing.T) { + statuses := []agentArtifactStatus{ + { + agent: testAgent(), + sources: []agentSourceProbe{ + {label: "GitHub release asset service-redis_0.0.74_linux_amd64.tar.gz", detail: "404"}, + {label: "OCI registry", detail: "not configured (set AGENT_REGISTRY)"}, + }, + }, + } + report := formatUnpublishedReport(statuses) + for _, want := range []string{ + "1 agent pin not downloadable in CI", + "agent codefly.dev/redis:0.0.74 is not published", + "GitHub release asset service-redis_0.0.74_linux_amd64.tar.gz: 404", + "OCI registry: not configured (set AGENT_REGISTRY)", + "-> tag + release the agent, or set AGENT_REGISTRY", + } { + if !strings.Contains(report, want) { + t.Fatalf("report missing %q\n%s", want, report) + } + } +} + +func TestFormatUnpublishedReportPluralizes(t *testing.T) { + statuses := []agentArtifactStatus{ + {agent: testAgent(), sources: []agentSourceProbe{{label: "GitHub", detail: "404"}}}, + {agent: &resources.Agent{Kind: resources.ServiceAgent, Publisher: "codefly.dev", Name: "vault", Version: "0.0.15"}, sources: []agentSourceProbe{{label: "GitHub", detail: "404"}}}, + } + report := formatUnpublishedReport(statuses) + if !strings.HasPrefix(report, "2 agent pins not downloadable in CI") { + t.Fatalf("report = %q, want plural header", report) + } +} diff --git a/cmd/ci/run.go b/cmd/ci/run.go index b03ea5fc..87e07d2f 100644 --- a/cmd/ci/run.go +++ b/cmd/ci/run.go @@ -52,6 +52,9 @@ var RunCmd = &cobra.Command{ return err } return runWithCIReport(ctx, workspace, plan, "codefly ci run", func(reporter *CIReporter) error { + if err := validateAgentVersions(ctx, workspace, plan); err != nil { + return err + } suites := normalizeTestSuites(testSuites) for _, phase := range phases { if phase == "verify" {