Add sshfs exporter class

.github/workflows/docker.yml

@@ -27,7 +27,9 @@ jobs:
         docker tag labgrid-client ${{ secrets.DOCKERHUB_PREFIX }}client
         docker tag labgrid-exporter ${{ secrets.DOCKERHUB_PREFIX }}exporter
         docker tag labgrid-coordinator ${{ secrets.DOCKERHUB_PREFIX }}coordinator
+        docker tag labgrid-sftp-server ${{ secrets.DOCKERHUB_PREFIX }}sftp-server
         docker push ${{ secrets.DOCKERHUB_PREFIX }}client
         docker push ${{ secrets.DOCKERHUB_PREFIX }}exporter
         docker push ${{ secrets.DOCKERHUB_PREFIX }}coordinator
+        docker push ${{ secrets.DOCKERHUB_PREFIX }}sftp-server
         docker images

.github/workflows/reusable-unit-tests.yml

@@ -30,8 +30,11 @@ jobs:
           ${{ runner.os }}-pip-
     - name: Install system dependencies
       run: |
-        sudo apt-get install -yq libow-dev openssh-server openssh-client libsnappy-dev ncurses-term graphviz openocd
+        sudo apt-get install -yq libow-dev openssh-server openssh-client libsnappy-dev ncurses-term graphviz openocd sshfs podman
         sudo mkdir -p /var/cache/labgrid/runner && sudo chown runner /var/cache/labgrid/runner
+    - name: Build sftp server
+      run: |
+        podman build --target labgrid-sftp-server -t labgrid/sftp-server:latest -f dockerfiles/Dockerfile .
     - name: Prepare local SSH
       run: |
         # the default of 777 is too open for SSH

dockerfiles/Dockerfile

@@ -121,3 +121,10 @@ COPY --from=ser2net /opt/ser2net /
 VOLUME /opt/conf
 
 CMD ["/entrypoint.sh"]
+
+#
+# SFTP server
+#
+FROM alpine:3.17 as labgrid-sftp-server
+
+RUN apk add openssh-sftp-server

dockerfiles/build.sh

@@ -15,7 +15,7 @@ export DOCKER_BUILDKIT=1
 
 VERSION="$(python -m setuptools_scm)"
 
-for t in client exporter coordinator; do
+for t in client exporter coordinator sftp-server; do
     ${DOCKER} build --build-arg VERSION="$VERSION" \
         --target labgrid-${t} -t labgrid-${t} -f "${SCRIPT_DIR}/Dockerfile" .
 done

labgrid/util/sshfs.py

@@ -0,0 +1,152 @@
+import logging
+import os
+import subprocess
+import random
+import string
+import time
+import shlex
+from contextlib import contextmanager
+
+import attr
+
+from .ssh import sshmanager
+from .timeout import Timeout
+from ..driver.exception import ExecutionError
+from ..resource.common import Resource, NetworkResource
+
+
+DEFAULT_SFTP_SERVER = "podman run --rm -i --mount type=bind,source={path},target={path} labgrid/sftp-server:latest usr/lib/ssh/sftp-server -e {readonly}"
+DEFAULT_RO_OPT = "-R"
+
+
+@attr.s
+class SSHFsExport:
+    """
+    Exports a local directory to a remote device using "reverse" SSH FS
+    """
+
+    local_path = attr.ib(
+        validator=attr.validators.instance_of(str),
+        converter=lambda x: os.path.abspath(str(x)),
+    )
+    resource = attr.ib(
+        validator=attr.validators.instance_of(Resource),
+    )
+    readonly = attr.ib(
+        default=True,
+        validator=attr.validators.instance_of(bool),
+    )
+
+    def __attrs_post_init__(self):
+        if not os.path.isdir(self.local_path):
+            raise FileNotFoundError(f"Local directory {self.local_path} not found")
+        self.logger = logging.getLogger(f"{self}")
+
+    @contextmanager
+    def export(self, remote_path=None):
+        host = self.resource.host
+        conn = sshmanager.open(host)
+
+        # If no remote path was specified, mount on a temporary directory
+        if remote_path is None:
+            tmpname = "".join(random.choices(string.ascii_lowercase, k=10))
+            remote_path = f"/tmp/labgrid-sshfs/{tmpname}/"
+            conn.run_check(f"mkdir -p {remote_path}")
+
+        env = self.resource.target.env
+
+        if env is None:
+            sftp_server_opt = DEFAULT_SFTP_SERVER
+            ro_opt = DEFAULT_RO_OPT
+        else:
+            sftp_server_opt = env.config.get_option("sftp_server", DEFAULT_SFTP_SERVER)
+            ro_opt = env.config.get_option("sftp_server_readonly_opt", DEFAULT_RO_OPT)
+
+        sftp_command = shlex.split(
+            sftp_server_opt.format(
+                path=self.local_path,
+                uid=str(os.getuid()),
+                gid=str(os.getgid()),
+                readonly=ro_opt if self.readonly else "",
+            )
+        )
+
+        sshfs_command = conn.get_prefix() + [
+            "sshfs",
+            "-o",
+            "slave",
+            "-o",
+            "idmap=user",
+            f":{self.local_path}",
+            remote_path,
+        ]
+
+        self.logger.info(
+            "Running %s <-> %s", " ".join(sftp_command), " ".join(sshfs_command)
+        )
+
+        # Reverse sshfs requires that sftp-server running locally and sshfs
+        # running remotely each have their stdout connected to the others
+        # stdin. Connecting sftp-server stdout to sshfs stdin is done using
+        # Popen pipes, but in order to connect sshfs stdout to sftp-stdin, an
+        # external pipe is needed.
+        (rfd, wfd) = os.pipe2(os.O_CLOEXEC)
+        try:
+            with subprocess.Popen(
+                sftp_command,
+                stdout=subprocess.PIPE,
+                stdin=rfd,
+            ) as sftp_server, subprocess.Popen(
+                sshfs_command,
+                stdout=wfd,
+                stdin=sftp_server.stdout,
+            ) as sshfs:
+                # Close all file descriptor open in this process. This way, if
+                # either process exits, the other will get the EPIPE error and
+                # exit also. If this process doesn't close its copy of the
+                # descriptors, that won't happen
+                sftp_server.stdout.close()
+
+                os.close(rfd)
+                rfd = -1
+
+                os.close(wfd)
+                wfd = -1
+
+                # Wait until the mount point appears remotely
+                t = Timeout(30.0)
+
+                while not t.expired:
+                    (_, _, exitcode) = conn.run(f"mountpoint --quiet {remote_path}")
+
+                    if exitcode == 0:
+                        break
+
+                    if sshfs.poll() is not None:
+                        raise ExecutionError(
+                            "sshfs process exited with {sshfs.returncode}"
+                        )
+
+                    if sftp_server.poll() is not None:
+                        raise ExecutionError(
+                            "sftp process exited with {sftp_server.returncode}"
+                        )
+
+                    time.sleep(1)
+
+                if t.expired:
+                    raise TimeoutError("Timeout waiting for SSH fs to mount")
+
+                try:
+                    yield remote_path
+                finally:
+                    sshfs.terminate()
+                    sftp_server.terminate()
+
+        finally:
+            # Cleanup if not done already
+            if rfd >= 0:
+                os.close(rfd)
+
+            if wfd >= 0:
+                os.close(wfd)

tests/test_util.py

@@ -15,6 +15,7 @@
 from labgrid.util.ssh import ForwardError, SSHConnection, sshmanager
 from labgrid.util.proxy import proxymanager
 from labgrid.util.managedfile import ManagedFile
+from labgrid.util.sshfs import SSHFsExport
 from labgrid.driver.exception import ExecutionError
 from labgrid.resource.serialport import NetworkSerialPort
 from labgrid.resource.common import Resource, NetworkResource
@@ -352,6 +353,49 @@ def test_local_managedfile(target, tmpdir):
     assert hash == mf.get_hash()
     assert str(t) == mf.get_remote_path()
 
+@pytest.mark.localsshmanager
+def test_sshfs(target, tmpdir):
+    t = tmpdir.join("test")
+    t_data = """
+Test
+"""
+    t.write(t_data)
+
+    res = NetworkResource(target, "test", "localhost")
+    sshfs = SSHFsExport(tmpdir, res)
+
+    with sshfs.export() as remote_path:
+        remote_test = f"{remote_path}/test"
+        #assert os.path.exists(remote_test)
+        with open(remote_test, "r") as f:
+            remote_data = f.read()
+
+        assert remote_data == t_data
+
+        # Read-only by default
+        with pytest.raises(PermissionError):
+            with open(remote_test, "w") as f:
+                pass
+
+@pytest.mark.localsshmanager
+def test_sshfs_write(target, tmpdir):
+    t = tmpdir.join("test")
+    t.write(
+"""
+Test
+"""
+    )
+
+    res = NetworkResource(target, "test", "localhost")
+    sshfs = SSHFsExport(tmpdir, res, readonly=False)
+
+    with sshfs.export() as remote_path:
+        remote_test = f"{remote_path}test"
+        #assert os.path.exists(remote_test)
+        with open(remote_test, "w") as f:
+            f.write("Hello")
+
+    assert t.read() == "Hello"
 
 def test_find_dict():
     dict_a = {"a": {"a.a": {"a.a.a": "a.a.a_val"}}, "b": "b_val"}