pkg/hostagent: Use in-process SSH client on executing requirement scripts

.github/workflows/test.yml

@@ -220,32 +220,50 @@ jobs:
     timeout-minutes: 120
     steps:
     - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+    - name: Fetch homebrew-core commit messages
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      with:
+        # needed by ./hack/brew-install-version.sh
+        repository: homebrew/homebrew-core
+        path: homebrew-core
+        fetch-depth: 0
+        filter: tree:0
     - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00  # v6.0.0
       with:
         go-version: 1.25.x
     - name: Unit tests
       run: go test -v ./...
     - name: Make
       run: make
+    - name: "Inject `no_timer_check` to kernel cmdline"
+      # workaround to https://github.com/lima-vm/lima/issues/84
+      run: |
+        export PATH="$PWD/_output/bin:$PATH"
+        ./hack/inject-cmdline-to-template.sh _output/share/lima/templates/_images/ubuntu.yaml no_timer_check
     - name: Install
       run: sudo make install
     - name: Validate jsonschema
       run: make schema-limayaml.json
     - name: Validate templates
       # Can't validate base templates in `_default` because they have no images
       run: find -L templates -name '*.yaml' ! -path '*/_default/*' | xargs limactl validate
+    - name: Install test dependencies (QEMU 10.1.1)
+      run: |
+        brew install bash coreutils
+        # QEMU 10.1.2 seems to break on GitHub runners
+        # We revert back to 10.1.1, which seems to work fine
+        git config --global user.name "GitHub Actions Bot"
+        git config --global user.email "nobody@localhost"
+        ./hack/brew-install-version.sh qemu 10.1.1
     - name: Install test dependencies
       # QEMU:      required by Lima itself
       # bash:      required by test-templates.sh (OS version of bash is too old)
       # coreutils: required by test-templates.sh for the "timeout" command
       # w3m :      required by test-templates.sh for port forwarding tests
       # socat:     required by test-templates.sh for port forwarding tests
-      run: brew install qemu bash coreutils w3m socat
+      run: brew install bash coreutils w3m socat
     - name: "Adjust LIMACTL_CREATE_ARGS"
       run: echo "LIMACTL_CREATE_ARGS=${LIMACTL_CREATE_ARGS} --vm-type=qemu" >>$GITHUB_ENV
-    - name: "Inject `no_timer_check` to kernel cmdline"
-      # workaround to https://github.com/lima-vm/lima/issues/84
-      run: ./hack/inject-cmdline-to-template.sh templates/_images/ubuntu.yaml no_timer_check
     - name: Cache image used by default.yaml
       uses: ./.github/actions/setup_cache_for_template
       with:
@@ -421,24 +439,42 @@ jobs:
     timeout-minutes: 120
     steps:
     - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+    - name: Fetch homebrew-core commit messages
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      with:
+        # needed by ./hack/brew-install-version.sh
+        repository: homebrew/homebrew-core
+        path: homebrew-core
+        fetch-depth: 0
+        filter: tree:0
     - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00  # v6.0.0
       with:
         go-version: 1.25.x
     - name: Make
       run: make
+    - name: "Inject `no_timer_check` to kernel cmdline"
+      # workaround to https://github.com/lima-vm/lima/issues/84
+      run: |
+        export PATH="$PWD/_output/bin:$PATH"
+        ./hack/inject-cmdline-to-template.sh _output/share/lima/templates/_images/ubuntu.yaml no_timer_check
     - name: Install
       run: sudo make install
     - name: "Adjust LIMACTL_CREATE_ARGS"
       run: echo "LIMACTL_CREATE_ARGS=${LIMACTL_CREATE_ARGS} --vm-type=qemu --network=lima:shared" >>$GITHUB_ENV
-    - name: "Inject `no_timer_check` to kernel cmdline"
-      # workaround to https://github.com/lima-vm/lima/issues/84
-      run: ./hack/inject-cmdline-to-template.sh templates/_images/ubuntu.yaml no_timer_check
     - name: Cache image used by default .yaml
       uses: ./.github/actions/setup_cache_for_template
       with:
         template: templates/default.yaml
+    - name: Install test dependencies (QEMU 10.1.1)
+      run: |
+        brew install bash coreutils
+        # QEMU 10.1.2 seems to break on GitHub runners
+        # We revert back to 10.1.1, which seems to work fine
+        git config --global user.name "GitHub Actions Bot"
+        git config --global user.email "nobody@localhost"
+        ./hack/brew-install-version.sh qemu 10.1.1
     - name: Install test dependencies
-      run: brew install qemu bash coreutils w3m socat
+      run: brew install bash coreutils w3m socat
     - name: Install socket_vmnet
       env:
         SOCKET_VMNET_VERSION: v1.2.0

go.mod

@@ -117,7 +117,7 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
-	golang.org/x/crypto v0.43.0 // indirect
+	golang.org/x/crypto v0.43.0
 	golang.org/x/mod v0.29.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/term v0.36.0 // indirect

pkg/cidata/cidata.TEMPLATE.d/user-data

@@ -104,3 +104,11 @@ bootcmd:
     {{- end }}
   {{- end }}
 {{- end }}
+
+{{- if .SSHHostKeys }}
+ssh_keys:
+  {{- range $type, $key := .SSHHostKeys }}
+  {{ $type }}: |
+{{ indent 4 $key }}
+  {{- end }}
+{{- end }}

pkg/cidata/cidata.go

@@ -118,7 +118,7 @@ func setupEnv(instConfigEnv map[string]string, propagateProxyEnv bool, slirpGate
 	return env, nil
 }
 
-func templateArgs(ctx context.Context, bootScripts bool, instDir, name string, instConfig *limatype.LimaYAML, udpDNSLocalPort, tcpDNSLocalPort, vsockPort int, virtioPort string, noCloudInit, rosettaEnabled, rosettaBinFmt bool) (*TemplateArgs, error) {
+func templateArgs(ctx context.Context, bootScripts bool, instDir, name string, instConfig *limatype.LimaYAML, udpDNSLocalPort, tcpDNSLocalPort, vsockPort int, virtioPort string, noCloudInit, rosettaEnabled, rosettaBinFmt, hostKeys bool) (*TemplateArgs, error) {
 	if err := limayaml.Validate(instConfig, false); err != nil {
 		return nil, err
 	}
@@ -342,11 +342,19 @@ func templateArgs(ctx context.Context, bootScripts bool, instDir, name string, i
 		}
 	}
 
+	if hostKeys {
+		sshHostKeys, err := sshutil.GenerateSSHHostKeys(instDir, args.Hostname)
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate SSH host keys: %w", err)
+		}
+		args.SSHHostKeys = sshHostKeys
+	}
+
 	return &args, nil
 }
 
 func GenerateCloudConfig(ctx context.Context, instDir, name string, instConfig *limatype.LimaYAML) error {
-	args, err := templateArgs(ctx, false, instDir, name, instConfig, 0, 0, 0, "", false, false, false)
+	args, err := templateArgs(ctx, false, instDir, name, instConfig, 0, 0, 0, "", false, false, false, false)
 	if err != nil {
 		return err
 	}
@@ -369,7 +377,7 @@ func GenerateCloudConfig(ctx context.Context, instDir, name string, instConfig *
 }
 
 func GenerateISO9660(ctx context.Context, drv driver.Driver, instDir, name string, instConfig *limatype.LimaYAML, udpDNSLocalPort, tcpDNSLocalPort int, guestAgentBinary, nerdctlArchive string, vsockPort int, virtioPort string, noCloudInit, rosettaEnabled, rosettaBinFmt bool) error {
-	args, err := templateArgs(ctx, true, instDir, name, instConfig, udpDNSLocalPort, tcpDNSLocalPort, vsockPort, virtioPort, noCloudInit, rosettaEnabled, rosettaBinFmt)
+	args, err := templateArgs(ctx, true, instDir, name, instConfig, udpDNSLocalPort, tcpDNSLocalPort, vsockPort, virtioPort, noCloudInit, rosettaEnabled, rosettaBinFmt, true)
 	if err != nil {
 		return err
 	}
@@ -467,6 +475,13 @@ func GenerateISO9660(ctx context.Context, drv driver.Driver, instDir, name strin
 			Path:   "ssh_authorized_keys",
 			Reader: strings.NewReader(strings.Join(args.SSHPubKeys, "\n")),
 		})
+		for keyType, keyContent := range args.SSHHostKeys {
+			suffix := strings.Replace(strings.Replace(keyType, "_public", "_key.pub", 1), "_private", "_key", 1)
+			layout = append(layout, iso9660util.Entry{
+				Path:   "ssh_host_" + suffix,
+				Reader: strings.NewReader(keyContent),
+			})
+		}
 		return writeCIDataDir(filepath.Join(instDir, filenames.CIDataISODir), layout)
 	}
 

pkg/cidata/template.go

@@ -115,6 +115,7 @@ type TemplateArgs struct {
 	Plain                           bool
 	TimeZone                        string
 	NoCloudInit                     bool
+	SSHHostKeys                     map[string]string // `ssh_keys` field in cloud-init SSH module
 }
 
 func ValidateTemplateArgs(args *TemplateArgs) error {

pkg/driver/vz/vm_darwin.go

@@ -113,18 +113,18 @@ func startVM(ctx context.Context, inst *limatype.Instance, sshLocalPort int) (vm
 								useSSHOverVsock = b
 							}
 						}
+						hostAddress := net.JoinHostPort(inst.SSHAddress, strconv.Itoa(usernetSSHLocalPort))
 						if !useSSHOverVsock {
 							logrus.Info("LIMA_SSH_OVER_VSOCK is false, skipping detection of SSH server on vsock port")
-						} else if err := usernetClient.WaitOpeningSSHPort(ctx, inst); err == nil {
-							hostAddress := net.JoinHostPort(inst.SSHAddress, strconv.Itoa(usernetSSHLocalPort))
-							if err := wrapper.startVsockForwarder(ctx, 22, hostAddress); err == nil {
-								logrus.Infof("Detected SSH server is listening on the vsock port; changed %s to proxy for the vsock port", hostAddress)
-								usernetSSHLocalPort = 0 // disable gvisor ssh port forwarding
-							} else {
-								logrus.WithError(err).Warn("Failed to detect SSH server on vsock port, falling back to usernet forwarder")
-							}
+						} else if err := usernetClient.WaitOpeningSSHPort(ctx, inst); err != nil {
+							logrus.WithError(err).Info("Failed to wait for the guest SSH server to become available, falling back to usernet forwarder")
+						} else if err := wrapper.checkSSHOverVsockAvailable(ctx, inst); err != nil {
+							logrus.WithError(err).Info("Failed to detect SSH server on vsock port, falling back to usernet forwarder")
+						} else if err := wrapper.startVsockForwarder(ctx, 22, hostAddress); err != nil {
+							logrus.WithError(err).Info("Failed to start SSH server forwarder on vsock port, falling back to usernet forwarder")
 						} else {
-							logrus.WithError(err).Warn("Failed to wait for the guest SSH server to become available, falling back to usernet forwarder")
+							logrus.Infof("Detected SSH server is listening on the vsock port; changed %s to proxy for the vsock port", hostAddress)
+							usernetSSHLocalPort = 0 // disable gvisor ssh port forwarding
 						}
 						err := usernetClient.ConfigureDriver(ctx, inst, usernetSSHLocalPort)
 						if err != nil {

pkg/driver/vz/vsock_forwarder.go

@@ -12,17 +12,14 @@ import (
 
 	"github.com/containers/gvisor-tap-vsock/pkg/tcpproxy"
 	"github.com/sirupsen/logrus"
+
+	"github.com/lima-vm/lima/v2/pkg/limatype"
+	"github.com/lima-vm/lima/v2/pkg/sshutil"
 )
 
 func (m *virtualMachineWrapper) startVsockForwarder(ctx context.Context, vsockPort uint32, hostAddress string) error {
-	// Test if the vsock port is open
-	conn, err := m.dialVsock(ctx, vsockPort)
-	if err != nil {
-		return err
-	}
-	conn.Close()
 	// Start listening on localhost:hostPort and forward to vsock:vsockPort
-	_, _, err = net.SplitHostPort(hostAddress)
+	_, _, err := net.SplitHostPort(hostAddress)
 	if err != nil {
 		return err
 	}
@@ -73,3 +70,9 @@ func (m *virtualMachineWrapper) dialVsock(_ context.Context, port uint32) (conn
 	}
 	return nil, err
 }
+
+func (m *virtualMachineWrapper) checkSSHOverVsockAvailable(ctx context.Context, inst *limatype.Instance) error {
+	return sshutil.WaitSSHReady(ctx, func(ctx context.Context) (net.Conn, error) {
+		return m.dialVsock(ctx, uint32(22))
+	}, "vsock:22", *inst.Config.User.Name, inst.Name, 1)
+}

pkg/driver/wsl2/boot/02-no-cloud-init-setup.sh

@@ -17,6 +17,11 @@ chmod 700 "${LIMA_CIDATA_HOME}"/.ssh/
 cp "${LIMA_CIDATA_MNT}"/ssh_authorized_keys "${LIMA_CIDATA_HOME}"/.ssh/authorized_keys
 chown "${LIMA_CIDATA_UID}:${LIMA_CIDATA_GID}" "${LIMA_CIDATA_HOME}"/.ssh/authorized_keys
 chmod 600 "${LIMA_CIDATA_HOME}"/.ssh/authorized_keys
+# copy SSH host keys
+mkdir -p /etc/ssh/
+cp "${LIMA_CIDATA_MNT}"/ssh_host_* /etc/ssh/
+chmod 600 /etc/ssh/ssh_host_*
+chmod 644 /etc/ssh/ssh_host_*.pub
 
 # add $LIMA_CIDATA_USER to sudoers
 echo "${LIMA_CIDATA_USER} ALL=(ALL) NOPASSWD:ALL" | tee -a /etc/sudoers.d/99_lima_sudoers

pkg/hostagent/requirements.go

@@ -6,8 +6,11 @@ package hostagent
 import (
 	"errors"
 	"fmt"
+	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/lima-vm/sshocker/pkg/ssh"
@@ -103,39 +106,65 @@ func (a *HostAgent) waitForRequirement(r requirement) error {
 	if err != nil {
 		return err
 	}
+	var stdout, stderr string
 	sshConfig := a.sshConfig
-	if r.noMaster || runtime.GOOS == "windows" {
-		// Remove ControlMaster, ControlPath, and ControlPersist options,
-		// because Cygwin-based SSH clients do not support multiplexing when executing commands.
-		// References:
-		//   https://inbox.sourceware.org/cygwin/c98988a5-7e65-4282-b2a1-bb8e350d5fab@acm.org/T/
-		//   https://stackoverflow.com/questions/20959792/is-ssh-controlmaster-with-cygwin-on-windows-actually-possible
-		// By removing these options:
-		//   - Avoids execution failures when the control master is not yet available.
-		//   - Prevents error messages such as:
-		//     > mux_client_request_session: read from master failed: Connection reset by peer
-		//     > ControlSocket ....sock already exists, disabling multiplexing
-		//     > mm_send_fd: sendmsg(2): Connection reset by peer\\r\\nmux_client_request_session: send fds failed\\r\\n
-		sshConfig = &ssh.SSHConfig{
-			ConfigFile:     sshConfig.ConfigFile,
-			Persist:        false,
-			AdditionalArgs: sshutil.DisableControlMasterOptsFromSSHArgs(sshConfig.AdditionalArgs),
+	if r.external || determineUseExternalSSH() {
+		if r.noMaster || runtime.GOOS == "windows" {
+			// Remove ControlMaster, ControlPath, and ControlPersist options,
+			// because Cygwin-based SSH clients do not support multiplexing when executing commands.
+			// References:
+			//   https://inbox.sourceware.org/cygwin/c98988a5-7e65-4282-b2a1-bb8e350d5fab@acm.org/T/
+			//   https://stackoverflow.com/questions/20959792/is-ssh-controlmaster-with-cygwin-on-windows-actually-possible
+			// By removing these options:
+			//   - Avoids execution failures when the control master is not yet available.
+			//   - Prevents error messages such as:
+			//     > mux_client_request_session: read from master failed: Connection reset by peer
+			//     > ControlSocket ....sock already exists, disabling multiplexing
+			//     > mm_send_fd: sendmsg(2): Connection reset by peer\\r\\nmux_client_request_session: send fds failed\\r\\n
+			sshConfig = &ssh.SSHConfig{
+				ConfigFile:     sshConfig.ConfigFile,
+				Persist:        false,
+				AdditionalArgs: sshutil.DisableControlMasterOptsFromSSHArgs(sshConfig.AdditionalArgs),
+			}
 		}
+		stdout, stderr, err = ssh.ExecuteScript(a.instSSHAddress, a.sshLocalPort, sshConfig, script, r.description)
+	} else {
+		stdout, stderr, err = sshutil.ExecuteScriptViaInProcessClient(a.instSSHAddress, a.sshLocalPort, *a.instConfig.User.Name, a.instName, script, r.description)
 	}
-	stdout, stderr, err := ssh.ExecuteScript(a.instSSHAddress, a.sshLocalPort, sshConfig, script, r.description)
 	logrus.Debugf("stdout=%q, stderr=%q, err=%v", stdout, stderr, err)
 	if err != nil {
 		return fmt.Errorf("stdout=%q, stderr=%q: %w", stdout, stderr, err)
 	}
 	return nil
 }
 
+var determineUseExternalSSH = sync.OnceValue(func() bool {
+	var useExternalSSH bool
+	// allow overriding via LIMA_EXTERNAL_SSH_REQUIREMENT environment variable
+	if envVar := os.Getenv("LIMA_EXTERNAL_SSH_REQUIREMENT"); envVar != "" {
+		if b, err := strconv.ParseBool(envVar); err != nil {
+			logrus.WithError(err).Warnf("invalid LIMA_EXTERNAL_SSH_REQUIREMENT value %q", envVar)
+		} else {
+			useExternalSSH = b
+		}
+	}
+	if useExternalSSH {
+		logrus.Info("using external ssh command for executing requirement scripts")
+	} else {
+		logrus.Info("using in-process ssh client for executing requirement scripts")
+	}
+	return useExternalSSH
+})
+
 type requirement struct {
 	description string
 	script      string
 	debugHint   string
 	fatal       bool
 	noMaster    bool
+	// Execute the script externally via the ssh command instead of using the in-process client.
+	// noMaster will be ignored if external is false.
+	external bool
 }
 
 func (a *HostAgent) essentialRequirements() []requirement {
@@ -158,6 +187,7 @@ If any private key under ~/.ssh is protected with a passphrase, you need to have
 true
 `,
 		debugHint: `The persistent ssh ControlMaster should be started immediately.`,
+		external:  true,
 	}
 	if *a.instConfig.Plain {
 		req = append(req, startControlMasterReq)

pkg/limatype/filenames/filenames.go

@@ -50,6 +50,7 @@ const (
 	SerialVirtioSock        = "serialv.sock"
 	SSHSock                 = "ssh.sock"
 	SSHConfig               = "ssh.config"
+	SSHKnownHosts           = "ssh_known_hosts"
 	VhostSock               = "virtiofsd-%d.sock"
 	VNCDisplayFile          = "vncdisplay"
 	VNCPasswordFile         = "vncpassword"

pkg/networks/usernet/client.go

@@ -140,8 +140,10 @@ func (c *Client) WaitOpeningSSHPort(ctx context.Context, inst *limatype.Instance
 	if err != nil {
 		return err
 	}
+	user := *inst.Config.User.Name
+	instanceName := inst.Name
 	// -1 avoids both sides timing out simultaneously.
-	u := fmt.Sprintf("%s/extension/wait_port?ip=%s&port=22&timeout=%d", c.base, ipAddr, timeoutSeconds-1)
+	u := fmt.Sprintf("%s/extension/wait-ssh-server?ip=%s&port=22&timeout=%d&user=%s&instance-name=%s", c.base, ipAddr, timeoutSeconds-1, user, instanceName)
 	res, err := httpclientutil.Get(ctx, c.client, u)
 	if err != nil {
 		return err