hostagent: gate Ready on raw-TCP SSH banner probe

The host agent fires its Ready event after essentialRequirements is
satisfied. Today the first SSH-related check runs a no-op script via
Lima's own SSH client, which has internal retry-on-failure behavior.
That makes the check coincidentally correct rather than a direct
statement about whether external clients can use the forwarded SSH port.

There is a race during VM bring-up where the host agent starts accepting
TCP connections on 127.0.0.1:<sshLocalPort> before guest sshd has bound
:22 inside the guest. A fresh external connection in that window gets
accepted, the proxy has no live peer, the host side closes, and the
client's first write fails with EPIPE ("Broken pipe").

The race is consistently reproducible: probing the forwarded port with
ssh-keyscan during `limactl start` shows 7-8s of EPIPE on restart
(vz, macOS aarch64, Ubuntu 24.04 cloudimg) and ~11s on first boot. On
well-provisioned hardware the subsequent requirements (user session,
ControlMaster, final boot scripts) coincidentally wait long enough that
the race has closed before Ready fires. With --plain mode, non-Linux
guests (which return immediately after the ssh requirement), or slower
hardware, the race can extend past `limactl start` returning, and
downstream tools that invoke ssh-keyscan / sftp / rsync immediately
after see EPIPE.

This commit adds a new essential requirement, "sshLocalPort serves an
SSH banner", at the head of the list. It opens a fresh raw TCP
connection to 127.0.0.1:<sshLocalPort>, reads the SSH identification
string per RFC4253 §4.2, and validates the prefix. No SSH client is
involved, so there is no internal retry that could mask the race.
waitForRequirements wraps the probe in its existing 3-second backoff
retry loop, so a transient EPIPE during bring-up just causes another
attempt rather than failing the start.

To support host-side native checks alongside the existing script-based
ones, the requirement struct gains a `check func(ctx) error` field;
waitForRequirement dispatches between the two (mutually exclusive per
requirement). context is now threaded through waitForRequirements so
the retry loop honors cancellation.

Pure-Go unit tests cover banner success, the SSH-1.99 legacy prefix,
accept-then-close (the exact race shape), wrong banner, no listener,
and a hung-write read-deadline case.

Signed-off-by: Weishi Z <amwish.zeng@gmail.com>

Author: WeishiZ

pkg/hostagent/hostagent.go

@@ -557,7 +557,7 @@ func (a *HostAgent) startHostAgentRoutines(ctx context.Context) error {
 		return nil
 	})
 	var errs []error
-	if err := a.waitForRequirements("essential", a.essentialRequirements()); err != nil {
+	if err := a.waitForRequirements(ctx, "essential", a.essentialRequirements()); err != nil {
 		errs = append(errs, err)
 	}
 	if *a.instConfig.SSH.ForwardAgent {
@@ -629,7 +629,7 @@ sudo chown -R "${USER}" /run/host-services`
 			}()
 		}
 	}
-	if err := a.waitForRequirements("optional", a.optionalRequirements()); err != nil {
+	if err := a.waitForRequirements(ctx, "optional", a.optionalRequirements()); err != nil {
 		errs = append(errs, err)
 	}
 	if hasGuestAgentDaemon {
@@ -641,7 +641,7 @@ sudo chown -R "${USER}" /run/host-services`
 			errs = append(errs, errors.New("guest agent does not seem to be running; port forwards will not work"))
 		}
 	}
-	if err := a.waitForRequirements("final", a.finalRequirements()); err != nil {
+	if err := a.waitForRequirements(ctx, "final", a.finalRequirements()); err != nil {
 		errs = append(errs, err)
 	}
 	// Copy all config files _after_ the requirements are done

pkg/hostagent/requirements.go

@@ -4,8 +4,11 @@
 package hostagent
 
 import (
+	"bufio"
+	"context"
 	"errors"
 	"fmt"
+	"net"
 	"runtime"
 	"strings"
 	"time"
@@ -17,7 +20,7 @@ import (
 	"github.com/lima-vm/lima/v2/pkg/sshutil"
 )
 
-func (a *HostAgent) waitForRequirements(label string, requirements []requirement) error {
+func (a *HostAgent) waitForRequirements(ctx context.Context, label string, requirements []requirement) error {
 	const (
 		retries       = 200
 		sleepDuration = 3 * time.Second
@@ -28,7 +31,7 @@ func (a *HostAgent) waitForRequirements(label string, requirements []requirement
 		logrus.Infof("Waiting for the %s requirement %d of %d: %q", label, i+1, len(requirements), req.description)
 	retryLoop:
 		for j := range retries {
-			err := a.waitForRequirement(req)
+			err := a.waitForRequirement(ctx, req)
 			if err == nil {
 				logrus.Infof("The %s requirement %d of %d is satisfied", label, i+1, len(requirements))
 				break retryLoop
@@ -42,7 +45,12 @@ func (a *HostAgent) waitForRequirements(label string, requirements []requirement
 				errs = append(errs, fmt.Errorf("failed to satisfy the %s requirement %d of %d %q: %s: %w", label, i+1, len(requirements), req.description, req.debugHint, err))
 				break retryLoop
 			}
-			time.Sleep(sleepDuration)
+			select {
+			case <-ctx.Done():
+				errs = append(errs, fmt.Errorf("failed to satisfy the %s requirement %d of %d %q: %s: %w", label, i+1, len(requirements), req.description, req.debugHint, ctx.Err()))
+				return errors.Join(errs...)
+			case <-time.After(sleepDuration):
+			}
 		}
 	}
 	return errors.Join(errs...)
@@ -109,7 +117,11 @@ func (a *HostAgent) bashAvailable() bool {
 	return *a.instConfig.OS != limatype.FREEBSD
 }
 
-func (a *HostAgent) waitForRequirement(r requirement) error {
+func (a *HostAgent) waitForRequirement(ctx context.Context, r requirement) error {
+	if r.check != nil {
+		logrus.Debugf("checking requirement %q", r.description)
+		return r.check(ctx)
+	}
 	logrus.Debugf("executing script %q", r.description)
 	script := r.script
 	if a.bashAvailable() {
@@ -147,16 +159,40 @@ func (a *HostAgent) waitForRequirement(r requirement) error {
 	return nil
 }
 
+// requirement describes a single readiness check performed by the host agent
+// before firing the Ready event. Exactly one of script and check must be set:
+//   - script: a shell script executed inside the guest via ssh.ExecuteScript.
+//   - check:  a host-side Go probe that returns nil when the requirement is satisfied.
 type requirement struct {
 	description string
 	script      string
+	check       func(ctx context.Context) error
 	debugHint   string
 	fatal       bool
 	noMaster    bool
 }
 
 func (a *HostAgent) essentialRequirements() []requirement {
 	req := make([]requirement, 0)
+	// Probe the host-facing forwarded SSH port with a raw TCP connection before
+	// running the script-based "ssh" requirement below. The script path uses
+	// Lima's own SSH client, which has internal retry behavior that can mask a
+	// race where the hostagent accepts a TCP connection on 127.0.0.1:sshLocalPort
+	// before guest sshd has bound :22. External clients (ssh-keyscan, sftp,
+	// rsync, ...) that connect during that window get EPIPE on the first write.
+	// This native probe proves the public-facing path is end-to-end usable.
+	port := a.sshLocalPort
+	req = append(req,
+		requirement{
+			description: "sshLocalPort serves an SSH banner",
+			check: func(ctx context.Context) error {
+				return probeSSHBannerOnLocalPort(ctx, port)
+			},
+			debugHint: `The host agent is listening on 127.0.0.1:<sshLocalPort> but the
+proxied connection into the guest is failing. Most commonly this means
+guest sshd has not yet bound :22. Check the serial log for sshd startup.
+`,
+		})
 	req = append(req,
 		requirement{
 			description: "ssh",
@@ -292,3 +328,41 @@ Check "` + logLocation + `" to see where the process is blocked!
 		})
 	return req
 }
+
+// probeSSHBannerTimeout bounds a single attempt of probeSSHBannerOnLocalPort.
+// waitForRequirements wraps the probe in its own retry loop (3s backoff), so
+// the per-attempt deadline only needs to be long enough to distinguish a
+// healthy SSH server from a hung TCP proxy.
+const probeSSHBannerTimeout = 5 * time.Second
+
+// probeSSHBannerOnLocalPort confirms that the host agent's TCP forwarder on
+// 127.0.0.1:port is serving an SSH banner end-to-end. The hostagent starts
+// accepting connections on 127.0.0.1:sshLocalPort before guest sshd is
+// necessarily ready on :22, so a fresh external connect+read may fail with
+// EPIPE during the bring-up window. This probe verifies the public-facing
+// path is healthy without using Lima's own SSH client (which can mask the
+// race via internal retries).
+func probeSSHBannerOnLocalPort(ctx context.Context, port int) error {
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
+	d := net.Dialer{Timeout: probeSSHBannerTimeout}
+	conn, err := d.DialContext(ctx, "tcp", addr)
+	if err != nil {
+		return fmt.Errorf("dial %s: %w", addr, err)
+	}
+	defer conn.Close()
+	if err := conn.SetReadDeadline(time.Now().Add(probeSSHBannerTimeout)); err != nil {
+		return fmt.Errorf("set read deadline on %s: %w", addr, err)
+	}
+	// RFC4253 §4.2: an SSH server sends an identification string ending in CR
+	// LF (or just LF historically) before the version exchange. If the proxied
+	// connection inside the guest has no live peer, the host side will close
+	// and ReadString returns EOF.
+	banner, err := bufio.NewReader(conn).ReadString('\n')
+	if err != nil {
+		return fmt.Errorf("read SSH banner from %s: %w", addr, err)
+	}
+	if !strings.HasPrefix(banner, "SSH-2.0-") && !strings.HasPrefix(banner, "SSH-1.99-") {
+		return fmt.Errorf("unexpected banner from %s: %q", addr, strings.TrimRight(banner, "\r\n"))
+	}
+	return nil
+}

pkg/hostagent/requirements_test.go

@@ -0,0 +1,120 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package hostagent
+
+import (
+	"context"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"gotest.tools/v3/assert"
+)
+
+// startBannerServer starts a TCP listener on 127.0.0.1:0 that emits the given
+// banner to each accepted connection and then closes it. If banner is empty,
+// the connection is closed immediately (simulating the hostagent-proxies-to-
+// dead-guest-sshd race).
+func startBannerServer(t *testing.T, banner string) int {
+	t.Helper()
+	var lc net.ListenConfig
+	ln, err := lc.Listen(t.Context(), "tcp", "127.0.0.1:0")
+	assert.NilError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go func(c net.Conn) {
+				defer c.Close()
+				if banner != "" {
+					_, _ = c.Write([]byte(banner))
+				}
+			}(conn)
+		}
+	}()
+	_, portStr, err := net.SplitHostPort(ln.Addr().String())
+	assert.NilError(t, err)
+	port, err := strconv.Atoi(portStr)
+	assert.NilError(t, err)
+	return port
+}
+
+func TestProbeSSHBannerOnLocalPort_OK(t *testing.T) {
+	port := startBannerServer(t, "SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13\r\n")
+	assert.NilError(t, probeSSHBannerOnLocalPort(t.Context(), port))
+}
+
+func TestProbeSSHBannerOnLocalPort_LegacyBanner(t *testing.T) {
+	// RFC4253 §5.1 reserves "SSH-1.99-" for servers that support both v1 and v2.
+	port := startBannerServer(t, "SSH-1.99-OpenSSH_old\r\n")
+	assert.NilError(t, probeSSHBannerOnLocalPort(t.Context(), port))
+}
+
+func TestProbeSSHBannerOnLocalPort_AcceptThenClose(t *testing.T) {
+	// Simulates the race: TCP forwarder accepts but the proxied guest peer is
+	// not listening, so the host side closes immediately and the client sees EOF.
+	port := startBannerServer(t, "")
+	err := probeSSHBannerOnLocalPort(t.Context(), port)
+	assert.ErrorContains(t, err, "read SSH banner")
+}
+
+func TestProbeSSHBannerOnLocalPort_WrongBanner(t *testing.T) {
+	port := startBannerServer(t, "HTTP/1.1 200 OK\r\n")
+	err := probeSSHBannerOnLocalPort(t.Context(), port)
+	assert.ErrorContains(t, err, "unexpected banner")
+}
+
+func TestProbeSSHBannerOnLocalPort_NoListener(t *testing.T) {
+	// Bind a listener to grab a free port, then close it before probing so the
+	// dial returns ECONNREFUSED (or equivalent on the host platform).
+	var lc net.ListenConfig
+	ln, err := lc.Listen(t.Context(), "tcp", "127.0.0.1:0")
+	assert.NilError(t, err)
+	_, portStr, err := net.SplitHostPort(ln.Addr().String())
+	assert.NilError(t, err)
+	port, err := strconv.Atoi(portStr)
+	assert.NilError(t, err)
+	assert.NilError(t, ln.Close())
+
+	err = probeSSHBannerOnLocalPort(t.Context(), port)
+	assert.Assert(t, err != nil)
+	assert.Assert(t, strings.Contains(err.Error(), "dial 127.0.0.1:"+portStr))
+}
+
+func TestProbeSSHBannerOnLocalPort_HungWrite(t *testing.T) {
+	// Server accepts but neither writes a banner nor closes, so the probe must
+	// time out on the read deadline rather than hang forever.
+	var lc net.ListenConfig
+	ln, err := lc.Listen(t.Context(), "tcp", "127.0.0.1:0")
+	assert.NilError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+	accepted := make(chan net.Conn, 1)
+	go func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			return
+		}
+		accepted <- conn
+	}()
+	_, portStr, err := net.SplitHostPort(ln.Addr().String())
+	assert.NilError(t, err)
+	port, err := strconv.Atoi(portStr)
+	assert.NilError(t, err)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 2*probeSSHBannerTimeout)
+	defer cancel()
+	start := time.Now()
+	err = probeSSHBannerOnLocalPort(ctx, port)
+	elapsed := time.Since(start)
+	if conn := <-accepted; conn != nil {
+		_ = conn.Close()
+	}
+	assert.Assert(t, err != nil)
+	assert.Assert(t, elapsed < 2*probeSSHBannerTimeout, "probe did not honor read deadline: elapsed=%s", elapsed)
+}