Implement unix socket forwarding

examples/docker.yaml

@@ -2,13 +2,7 @@
 # $ limactl start ./docker.yaml
 # $ limactl shell docker docker run -it -v $HOME:$HOME --rm alpine
 
-# Hint: To allow `docker` CLI on the host to connect to the Docker daemon running inside the guest,
-# add `NoHostAuthenticationForLocalhost yes` in ~/.ssh/config , and then run the following commands:
-# $ export DOCKER_HOST=ssh://localhost:60006
-# $ docker ...
-
-# If ssh:// ... does not work, try the following commands:
-# $ ssh -f -N -p 60006 -i ~/.lima/_config/user -o NoHostAuthenticationForLocalhost=yes -L $HOME/docker.sock:/run/user/$(id -u)/docker.sock 127.0.0.1
+# To run `docker` on the host (assumes docker-cli is installed):
 # $ export DOCKER_HOST=unix://$HOME/docker.sock
 # $ docker ...
 
@@ -63,3 +57,6 @@ probes:
         exit 1
       fi
     hint: See "/var/log/cloud-init-output.log". in the guest
+portForwards:
+  - guestSocket: "/run/user/{{.UID}}/docker.sock"
+    hostSocket: "{{.Home}}/docker.sock"

examples/podman.yaml

@@ -2,14 +2,11 @@
 # $ limactl start ./podman.yaml
 # $ limactl shell podman podman run -it --rm -v $HOME:$HOME:ro docker.io/library/alpine
 
-# Hint: To allow `podman` CLI on the host to connect to the Podman daemon running inside the guest,
-# add `NoHostAuthenticationForLocalhost yes` in ~/.ssh/config , and then run the following commands:
-# $ export CONTAINER_HOST=ssh://$(id -un)@localhost:60906/run/user/$(id -u)/podman/podman.sock
-# $ export CONTAINER_SSHKEY=$HOME/.lima/_config/user
+# Hint: To allow `podman` CLI on the host to connect to the Podman daemon running inside the guest, run the following commands:
+# $ export CONTAINER_HOST=unix://$HOME/podman.sock
 # $ podman ...
 
 # Hint: To allow `docker` CLI on the host to connect to the Podman daemon running inside the guest, run the following commands:
-# $ ssh -f -N -p 60906 -i ~/.lima/_config/user -o NoHostAuthenticationForLocalhost=yes -L $HOME/podman.sock:/run/user/$(id -u)/podman/podman.sock 127.0.0.1
 # $ export DOCKER_HOST=unix://$HOME/podman.sock
 # $ docker ...
 
@@ -52,3 +49,6 @@ probes:
         exit 1
       fi
     hint: See "/var/log/cloud-init-output.log". in the guest
+portForwards:
+  - guestSocket: "/run/user/{{.UID}}/podman/podman.sock"
+    hostSocket: "{{.Home}}/podman.sock"

pkg/hostagent/hostagent.go

@@ -4,13 +4,15 @@ import (
 	"bufio"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -127,13 +129,13 @@ func New(instName string, stdout io.Writer, sigintCh chan os.Signal, opts ...Opt
 	// Block ports 22 and sshLocalPort on all IPs
 	for _, port := range []int{sshGuestPort, sshLocalPort} {
 		rule := limayaml.PortForward{GuestIP: net.IPv4zero, GuestPort: port, Ignore: true}
-		limayaml.FillPortForwardDefaults(&rule)
+		limayaml.FillPortForwardDefaults(&rule, inst.Dir)
 		rules = append(rules, rule)
 	}
 	rules = append(rules, y.PortForwards...)
 	// Default forwards for all non-privileged ports from "127.0.0.1" and "::1"
 	rule := limayaml.PortForward{GuestIP: guestagentapi.IPv4loopback1}
-	limayaml.FillPortForwardDefaults(&rule)
+	limayaml.FillPortForwardDefaults(&rule, inst.Dir)
 	rules = append(rules, rule)
 
 	a := &HostAgent{
@@ -281,9 +283,10 @@ func (a *HostAgent) Run(ctx context.Context) error {
 	stBooting := stBase
 	a.emitEvent(ctx, events.Event{Status: stBooting})
 
+	ctxHA, cancelHA := context.WithCancel(ctx)
 	go func() {
 		stRunning := stBase
-		if haErr := a.startHostAgentRoutines(ctx); haErr != nil {
+		if haErr := a.startHostAgentRoutines(ctxHA); haErr != nil {
 			stRunning.Degraded = true
 			stRunning.Errors = append(stRunning.Errors, haErr.Error())
 		}
@@ -295,12 +298,15 @@ func (a *HostAgent) Run(ctx context.Context) error {
 		select {
 		case <-a.sigintCh:
 			logrus.Info("Received SIGINT, shutting down the host agent")
+			cancelHA()
 			if closeErr := a.close(); closeErr != nil {
 				logrus.WithError(closeErr).Warn("an error during shutting down the host agent")
 			}
 			return a.shutdownQEMU(ctx, 3*time.Minute, qCmd, qWaitCh)
 		case qWaitErr := <-qWaitCh:
 			logrus.WithError(qWaitErr).Info("QEMU has exited")
+			// lint insists that we need to call cancelHA() on all possible codepaths
+			cancelHA()
 			return qWaitErr
 		}
 	}
@@ -400,33 +406,47 @@ func (a *HostAgent) close() error {
 func (a *HostAgent) watchGuestAgentEvents(ctx context.Context) {
 	// TODO: use vSock (when QEMU for macOS gets support for vSock)
 
+	// Setup all socket forwards and defer their teardown
+	logrus.Debugf("Forwarding unix sockets")
+	for _, rule := range a.y.PortForwards {
+		if rule.GuestSocket != "" {
+			local := hostAddress(rule, guestagentapi.IPPort{})
+			_ = forwardSSH(ctx, a.sshConfig, a.sshLocalPort, local, rule.GuestSocket, verbForward)
+		}
+	}
+
 	localUnix := filepath.Join(a.instDir, filenames.GuestAgentSock)
-	// guest should have same UID as the host (specified in cidata)
 	remoteUnix := "/run/lima-guestagent.sock"
 
+	a.onClose = append(a.onClose, func() error {
+		logrus.Debugf("Stop forwarding unix sockets")
+		var mErr error
+		for _, rule := range a.y.PortForwards {
+			if rule.GuestSocket != "" {
+				local := hostAddress(rule, guestagentapi.IPPort{})
+				// using ctx.Background() because ctx has already been cancelled
+				if err := forwardSSH(context.Background(), a.sshConfig, a.sshLocalPort, local, rule.GuestSocket, verbCancel); err != nil {
+					mErr = multierror.Append(mErr, err)
+				}
+			}
+		}
+		if err := forwardSSH(context.Background(), a.sshConfig, a.sshLocalPort, localUnix, remoteUnix, verbCancel); err != nil {
+			mErr = multierror.Append(mErr, err)
+		}
+		return mErr
+	})
+
 	for {
 		if !isGuestAgentSocketAccessible(ctx, localUnix) {
-			if err := os.RemoveAll(localUnix); err != nil {
-				logrus.WithError(err).Warnf("failed to clean up %q (host) before setting up forwarding", localUnix)
-			}
-			logrus.Infof("Forwarding %q (guest) to %q (host)", remoteUnix, localUnix)
-			if err := forwardSSH(ctx, a.sshConfig, a.sshLocalPort, localUnix, remoteUnix, false); err != nil {
-				logrus.WithError(err).Warnf("failed to setting up forward from %q (guest) to %q (host)", remoteUnix, localUnix)
-			}
+			_ = forwardSSH(ctx, a.sshConfig, a.sshLocalPort, localUnix, remoteUnix, verbForward)
 		}
 		if err := a.processGuestAgentEvents(ctx, localUnix); err != nil {
-			logrus.WithError(err).Warn("connection to the guest agent was closed unexpectedly")
+			if !errors.Is(err, context.Canceled) {
+				logrus.WithError(err).Warn("connection to the guest agent was closed unexpectedly")
+			}
 		}
 		select {
 		case <-ctx.Done():
-			logrus.Infof("Stopping forwarding %q to %q", remoteUnix, localUnix)
-			verbCancel := true
-			if err := forwardSSH(ctx, a.sshConfig, a.sshLocalPort, localUnix, remoteUnix, verbCancel); err != nil {
-				logrus.WithError(err).Warnf("failed to stop forwarding %q (remote) to %q (local)", remoteUnix, localUnix)
-			}
-			if err := os.RemoveAll(localUnix); err != nil {
-				logrus.WithError(err).Warnf("failed to clean up %q (host) after stopping forwarding", localUnix)
-			}
 			return
 		case <-time.After(10 * time.Second):
 		}
@@ -469,12 +489,13 @@ func (a *HostAgent) processGuestAgentEvents(ctx context.Context, localUnix strin
 	return io.EOF
 }
 
-func forwardSSH(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local, remote string, cancel bool) error {
+const (
+	verbForward = "forward"
+	verbCancel  = "cancel"
+)
+
+func forwardSSH(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local, remote string, verb string) error {
 	args := sshConfig.Args()
-	verb := "forward"
-	if cancel {
-		verb = "cancel"
-	}
 	args = append(args,
 		"-T",
 		"-O", verb,
@@ -485,8 +506,35 @@ func forwardSSH(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local,
 		"127.0.0.1",
 		"--",
 	)
+	if strings.HasPrefix(local, "/") {
+		switch verb {
+		case verbForward:
+			logrus.Infof("Forwarding %q (guest) to %q (host)", remote, local)
+			if err := os.RemoveAll(local); err != nil {
+				logrus.WithError(err).Warnf("Failed to clean up %q (host) before setting up forwarding", local)
+			}
+			if err := os.MkdirAll(filepath.Dir(local), 0750); err != nil {
+				return fmt.Errorf("can't create directory for local socket %q: %w", local, err)
+			}
+		case verbCancel:
+			logrus.Infof("Stopping forwarding %q (guest) to %q (host)", remote, local)
+			defer func() {
+				if err := os.RemoveAll(local); err != nil {
+					logrus.WithError(err).Warnf("Failed to clean up %q (host) after stopping forwarding", local)
+				}
+			}()
+		default:
+			panic(fmt.Errorf("invalid verb %q", verb))
+		}
+	}
 	cmd := exec.CommandContext(ctx, sshConfig.Binary(), args...)
 	if out, err := cmd.Output(); err != nil {
+		if verb == verbForward && strings.HasPrefix(local, "/") {
+			logrus.WithError(err).Warnf("Failed to set up forward from %q (guest) to %q (host)", remote, local)
+			if removeErr := os.RemoveAll(local); err != nil {
+				logrus.WithError(removeErr).Warnf("Failed to clean up %q (host) after forwarding failed", local)
+			}
+		}
 		return fmt.Errorf("failed to run %v: %q: %w", cmd.Args, string(out), err)
 	}
 	return nil

pkg/hostagent/port.go

@@ -13,7 +13,6 @@ import (
 type portForwarder struct {
 	sshConfig   *ssh.SSHConfig
 	sshHostPort int
-	tcp         map[int]struct{} // key: int (NOTE: this might be inconsistent with the actual status of SSH master)
 	rules       []limayaml.PortForward
 }
 
@@ -23,13 +22,29 @@ func newPortForwarder(sshConfig *ssh.SSHConfig, sshHostPort int, rules []limayam
 	return &portForwarder{
 		sshConfig:   sshConfig,
 		sshHostPort: sshHostPort,
-		tcp:         make(map[int]struct{}),
 		rules:       rules,
 	}
 }
 
+func hostAddress(rule limayaml.PortForward, guest api.IPPort) string {
+	if rule.HostSocket != "" {
+		return rule.HostSocket
+	}
+	host := api.IPPort{IP: rule.HostIP}
+	if guest.Port == 0 {
+		// guest is a socket
+		host.Port = rule.HostPort
+	} else {
+		host.Port = guest.Port + rule.HostPortRange[0] - rule.GuestPortRange[0]
+	}
+	return host.String()
+}
+
 func (pf *portForwarder) forwardingAddresses(guest api.IPPort) (string, string) {
 	for _, rule := range pf.rules {
+		if rule.GuestSocket != "" {
+			continue
+		}
 		if guest.Port < rule.GuestPortRange[0] || guest.Port > rule.GuestPortRange[1] {
 			continue
 		}
@@ -47,33 +62,21 @@ func (pf *portForwarder) forwardingAddresses(guest api.IPPort) (string, string)
 			}
 			break
 		}
-		host := api.IPPort{
-			IP:   rule.HostIP,
-			Port: guest.Port + rule.HostPortRange[0] - rule.GuestPortRange[0],
-		}
-		return host.String(), guest.String()
+		return hostAddress(rule, guest), guest.String()
 	}
 	return "", guest.String()
 }
 
 func (pf *portForwarder) OnEvent(ctx context.Context, ev api.Event) {
 	for _, f := range ev.LocalPortsRemoved {
-		// pf.tcp might be inconsistent with the actual state of the SSH master,
-		// so we always attempt to cancel forwarding, even when f.Port is not tracked in pf.tcp.
 		local, remote := pf.forwardingAddresses(f)
 		if local == "" {
 			continue
 		}
 		logrus.Infof("Stopping forwarding TCP from %s to %s", remote, local)
-		verbCancel := true
 		if err := forwardTCP(ctx, pf.sshConfig, pf.sshHostPort, local, remote, verbCancel); err != nil {
-			if _, ok := pf.tcp[f.Port]; ok {
-				logrus.WithError(err).Warnf("failed to stop forwarding TCP port %d", f.Port)
-			} else {
-				logrus.WithError(err).Debugf("failed to stop forwarding TCP port %d (negligible)", f.Port)
-			}
+			logrus.WithError(err).Warnf("failed to stop forwarding tcp port %d", f.Port)
 		}
-		delete(pf.tcp, f.Port)
 	}
 	for _, f := range ev.LocalPortsAdded {
 		local, remote := pf.forwardingAddresses(f)
@@ -82,10 +85,8 @@ func (pf *portForwarder) OnEvent(ctx context.Context, ev api.Event) {
 			continue
 		}
 		logrus.Infof("Forwarding TCP from %s to %s", remote, local)
-		if err := forwardTCP(ctx, pf.sshConfig, pf.sshHostPort, local, remote, false); err != nil {
-			logrus.WithError(err).Warnf("failed to set up forwarding TCP port %d (negligible if already forwarded)", f.Port)
-		} else {
-			pf.tcp[f.Port] = struct{}{}
+		if err := forwardTCP(ctx, pf.sshConfig, pf.sshHostPort, local, remote, verbForward); err != nil {
+			logrus.WithError(err).Warnf("failed to set up forwarding tcp port %d (negligible if already forwarded)", f.Port)
 		}
 	}
 }

pkg/hostagent/port_darwin.go

@@ -7,14 +7,19 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 
+	"github.com/lima-vm/lima/pkg/guestagent/api"
 	"github.com/lima-vm/sshocker/pkg/ssh"
 	"github.com/norouter/norouter/pkg/agent/bicopy"
 	"github.com/sirupsen/logrus"
 )
 
 // forwardTCP is not thread-safe
-func forwardTCP(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local, remote string, cancel bool) error {
+func forwardTCP(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local, remote string, verb string) error {
+	if strings.HasPrefix(local, "/") {
+		return forwardSSH(ctx, sshConfig, port, local, remote, verb)
+	}
 	localIPStr, localPortStr, err := net.SplitHostPort(local)
 	if err != nil {
 		return err
@@ -25,8 +30,8 @@ func forwardTCP(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local,
 		return err
 	}
 
-	if !net.ParseIP("127.0.0.1").Equal(localIP) || localPort >= 1024 {
-		return forwardSSH(ctx, sshConfig, port, local, remote, cancel)
+	if !localIP.Equal(api.IPv4loopback1) || localPort >= 1024 {
+		return forwardSSH(ctx, sshConfig, port, local, remote, verb)
 	}
 
 	// on macOS, listening on 127.0.0.1:80 requires root while 0.0.0.0:80 does not require root.
@@ -35,13 +40,13 @@ func forwardTCP(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local,
 	// We use "pseudoloopback" forwarder that listens on 0.0.0.0:80 but rejects connections from non-loopback src IP.
 	logrus.Debugf("using pseudoloopback port forwarder for %q", local)
 
-	if cancel {
+	if verb == verbCancel {
 		plf, ok := pseudoLoopbackForwarders[local]
 		if ok {
 			localUnix := plf.unixAddr.Name
 			_ = plf.Close()
 			delete(pseudoLoopbackForwarders, local)
-			if err := forwardSSH(ctx, sshConfig, port, localUnix, remote, cancel); err != nil {
+			if err := forwardSSH(ctx, sshConfig, port, localUnix, remote, verb); err != nil {
 				return err
 			}
 		} else {
@@ -56,18 +61,12 @@ func forwardTCP(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local,
 	}
 	localUnix := filepath.Join(localUnixDir, "sock")
 	logrus.Debugf("forwarding %q to %q", localUnix, remote)
-	if err := forwardSSH(ctx, sshConfig, port, localUnix, remote, cancel); err != nil {
-		if removeErr := os.RemoveAll(localUnixDir); removeErr != nil {
-			logrus.WithError(removeErr).Warnf("failed to remove %q", removeErr)
-		}
+	if err := forwardSSH(ctx, sshConfig, port, localUnix, remote, verb); err != nil {
 		return err
 	}
 	plf, err := newPseudoLoopbackForwarder(localPort, localUnix)
 	if err != nil {
-		if removeErr := os.RemoveAll(localUnixDir); removeErr != nil {
-			logrus.WithError(removeErr).Warnf("failed to remove %q", removeErr)
-		}
-		if cancelErr := forwardSSH(ctx, sshConfig, port, localUnix, remote, true); cancelErr != nil {
+		if cancelErr := forwardSSH(ctx, sshConfig, port, localUnix, remote, verbCancel); cancelErr != nil {
 			logrus.WithError(cancelErr).Warnf("failed to cancel forwarding %q to %q", localUnix, remote)
 		}
 		return err

pkg/hostagent/port_others.go

@@ -9,6 +9,6 @@ import (
 	"github.com/lima-vm/sshocker/pkg/ssh"
 )
 
-func forwardTCP(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local, remote string, cancel bool) error {
-	return forwardSSH(ctx, sshConfig, port, local, remote, cancel)
+func forwardTCP(ctx context.Context, sshConfig *ssh.SSHConfig, port int, local, remote string, verb string) error {
+	return forwardSSH(ctx, sshConfig, port, local, remote, verb)
 }