fix(provision/docker): probe host apiserver before declaring ready

pkg/provision/docker/docker.go

@@ -205,6 +205,20 @@ func Provision(ctx context.Context, cfg config.DockerConfig, logger *zap.Logger)
 	if err := kubecfg.Import(rawKubeconfig); err != nil {
 		return nil, fmt.Errorf("merge kubeconfig: %w", err)
 	}
+
+	// k3s.yaml landing in the container doesn't mean the host can
+	// reach the apiserver yet -- the docker port-forward to the
+	// host-mapped 6443 is bound a moment later, and the very next
+	// step (envoygateway.Install -> kubectl apply --server-side)
+	// would otherwise race against it with "dial tcp 127.0.0.1:
+	// 6443: connect: connection refused". Probe the host endpoint
+	// through the merged kubeconfig until /readyz succeeds before
+	// declaring readiness.
+	if err := c.waitForHostAPIServer(ctx); err != nil {
+		dlogs, _ := dockerexec.Logs(ctx, cli, cfg.Name, "100")
+		return nil, fmt.Errorf("wait for host apiserver: %w\ncontainer logs:\n%s", err, dlogs)
+	}
+
 	logger.Info("k3s ready", zap.String("context", cfg.Context))
 
 	// Install the bundled Envoy Gateway (CRDs + controller +
@@ -337,8 +351,11 @@ func (c *Cluster) NodeExec(ctx context.Context, command string, stdin io.Reader)
 }
 
 // waitForKubeconfig polls until the k3s-managed kubeconfig appears
-// inside the container. k3s writes /etc/rancher/k3s/k3s.yaml after
-// the apiserver is ready to accept connections.
+// inside the container. k3s writes /etc/rancher/k3s/k3s.yaml when
+// the in-container apiserver socket is bound; the host-side port
+// forward and full apiserver readiness lag behind, so the host
+// must additionally probe via waitForHostAPIServer before any
+// kubectl call against the merged kubeconfig.
 func (c *Cluster) waitForKubeconfig(ctx context.Context) error {
 	deadline := time.Now().Add(2 * time.Minute)
 	for {
@@ -357,6 +374,59 @@ func (c *Cluster) waitForKubeconfig(ctx context.Context) error {
 	}
 }
 
+const (
+	hostAPIServerReadyTimeout  = 60 * time.Second
+	hostAPIServerReadyInterval = time.Second
+)
+
+// waitForHostAPIServer polls `kubectl get --raw=/readyz` against
+// the merged context until the apiserver responds with a 200. Two
+// host-side concerns lag behind kubeconfig-in-container readiness:
+// docker's userland port forward to 127.0.0.1:HostAPIPort needs a
+// moment to bind, and the apiserver itself takes a beat to advance
+// from "listening" to "ready". /readyz covers both -- a connection
+// refused, a 503 from a still-starting apiserver, or a transport
+// error are all retried.
+//
+// We shell out to kubectl rather than dialing the apiserver
+// directly because envoygateway.Install drives the host kubeconfig
+// the same way -- using kubectl here keeps the readiness probe on
+// the same code path that the very next caller will use.
+func (c *Cluster) waitForHostAPIServer(ctx context.Context) error {
+	return c.pollHostAPIServerReadyz(ctx, hostAPIServerReadyTimeout, hostAPIServerReadyInterval)
+}
+
+// pollHostAPIServerReadyz is the parameterised body of
+// waitForHostAPIServer; the timeout and interval are arguments so
+// tests can drive the loop with a fake kubectl on $PATH at sub-second
+// resolution.
+func (c *Cluster) pollHostAPIServerReadyz(ctx context.Context, timeout, interval time.Duration) error {
+	c.logger.Info("waiting for host apiserver", zap.String("context", c.cfg.Context))
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for {
+		probe := exec.CommandContext(ctx, "kubectl",
+			"--context="+c.cfg.Context,
+			"get", "--raw=/readyz",
+		)
+		// Discard noisy intermediate failures; surface only the
+		// final state via lastErr if we time out.
+		out, err := probe.CombinedOutput()
+		if err == nil {
+			return nil
+		}
+		lastErr = fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out)))
+		if time.Now().After(deadline) {
+			return fmt.Errorf("apiserver /readyz never returned 200 within %s on context %q: %v", timeout, c.cfg.Context, lastErr)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(interval):
+		}
+	}
+}
+
 // extractKubeconfig reads the container's kubeconfig and rewrites
 // the embedded server URL to the host-mapped API port so the host's
 // kubectl can reach it.

pkg/provision/docker/docker_test.go

@@ -0,0 +1,97 @@
+package docker
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+
+	"github.com/Yolean/y-cluster/pkg/provision/config"
+)
+
+// fakeKubectlOnPATH writes an executable shell script named `kubectl`
+// to a fresh temp dir and prepends that dir to $PATH for the test.
+// pollHostAPIServerReadyz exec's `kubectl` by name, so the resolved
+// binary is the script rather than any real kubectl on the system.
+// `body` is the shell body (no shebang); use `exit 0` for the
+// success case and `exit 1` (with a stderr message) for failure.
+func fakeKubectlOnPATH(t *testing.T, body string) {
+	t.Helper()
+	if runtime.GOOS == "windows" {
+		t.Skip("fake kubectl shim is /bin/sh-only")
+	}
+	dir := t.TempDir()
+	script := "#!/bin/sh\n" + body + "\n"
+	if err := os.WriteFile(filepath.Join(dir, "kubectl"), []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
+}
+
+func newProbeTestCluster() *Cluster {
+	return &Cluster{
+		cfg: config.DockerConfig{
+			CommonConfig: config.CommonConfig{Context: "unit-test-ctx"},
+		},
+		logger: zap.NewNop(),
+	}
+}
+
+// First-call success: a kubectl that exits 0 returns nil immediately.
+func TestPollHostAPIServerReadyz_Success(t *testing.T) {
+	fakeKubectlOnPATH(t, "exit 0")
+	c := newProbeTestCluster()
+	if err := c.pollHostAPIServerReadyz(context.Background(), time.Second, 10*time.Millisecond); err != nil {
+		t.Fatalf("expected success, got: %v", err)
+	}
+}
+
+// Always-failing kubectl: deadline trips, the wrapped "never returned
+// 200" error is returned (not ctx.Err()) and carries the context name.
+func TestPollHostAPIServerReadyz_DeadlineHonored(t *testing.T) {
+	fakeKubectlOnPATH(t, `echo 'connection refused' >&2; exit 1`)
+	c := newProbeTestCluster()
+	start := time.Now()
+	err := c.pollHostAPIServerReadyz(context.Background(), 100*time.Millisecond, 20*time.Millisecond)
+	elapsed := time.Since(start)
+	if err == nil {
+		t.Fatal("expected deadline error, got nil")
+	}
+	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+		t.Fatalf("expected wrapped readiness error, got ctx error: %v", err)
+	}
+	if !strings.Contains(err.Error(), "/readyz never returned 200") {
+		t.Fatalf("expected readiness deadline message, got: %v", err)
+	}
+	if !strings.Contains(err.Error(), "unit-test-ctx") {
+		t.Fatalf("expected context name in error, got: %v", err)
+	}
+	// Sanity: we shouldn't have run anywhere near the production 60s.
+	if elapsed > 5*time.Second {
+		t.Fatalf("loop ran far longer than the test timeout: %s", elapsed)
+	}
+}
+
+// Caller-cancelled ctx: the loop returns ctx.Err() rather than the
+// readiness deadline message. Guards against a refactor that drops
+// the select { <-ctx.Done() } branch and silently makes the wait
+// non-cancellable.
+func TestPollHostAPIServerReadyz_ContextCanceled(t *testing.T) {
+	fakeKubectlOnPATH(t, `echo failing >&2; exit 1`)
+	c := newProbeTestCluster()
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	err := c.pollHostAPIServerReadyz(ctx, 10*time.Second, 5*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected ctx error, got nil")
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("expected context.DeadlineExceeded, got: %v", err)
+	}
+}