diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 13ee64ee7..d28b1df71 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,9 @@ jobs: docker tag labgrid-client ${{ secrets.DOCKERHUB_PREFIX }}client docker tag labgrid-exporter ${{ secrets.DOCKERHUB_PREFIX }}exporter docker tag labgrid-coordinator ${{ secrets.DOCKERHUB_PREFIX }}coordinator + docker tag labgrid-sftp-server ${{ secrets.DOCKERHUB_PREFIX }}sftp-server docker push ${{ secrets.DOCKERHUB_PREFIX }}client docker push ${{ secrets.DOCKERHUB_PREFIX }}exporter docker push ${{ secrets.DOCKERHUB_PREFIX }}coordinator + docker push ${{ secrets.DOCKERHUB_PREFIX }}sftp-server docker images diff --git a/.github/workflows/reusable-unit-tests.yml b/.github/workflows/reusable-unit-tests.yml index e1bcef82b..76a6c1d23 100644 --- a/.github/workflows/reusable-unit-tests.yml +++ b/.github/workflows/reusable-unit-tests.yml @@ -30,8 +30,11 @@ jobs: ${{ runner.os }}-pip- - name: Install system dependencies run: | - sudo apt-get install -yq libow-dev openssh-server openssh-client libsnappy-dev ncurses-term graphviz openocd + sudo apt-get install -yq libow-dev openssh-server openssh-client libsnappy-dev ncurses-term graphviz openocd sshfs podman sudo mkdir -p /var/cache/labgrid/runner && sudo chown runner /var/cache/labgrid/runner + - name: Build sftp server + run: | + podman build --target labgrid-sftp-server -t labgrid/sftp-server:latest -f dockerfiles/Dockerfile . - name: Prepare local SSH run: | # the default of 777 is too open for SSH diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 2e5182cea..00be3601c 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -121,3 +121,10 @@ COPY --from=ser2net /opt/ser2net / VOLUME /opt/conf CMD ["/entrypoint.sh"] + +# +# SFTP server +# +FROM alpine:3.17 as labgrid-sftp-server + +RUN apk add openssh-sftp-server diff --git a/dockerfiles/build.sh b/dockerfiles/build.sh index 356cb24a4..fd1bbf859 100755 --- a/dockerfiles/build.sh +++ b/dockerfiles/build.sh @@ -15,7 +15,7 @@ export DOCKER_BUILDKIT=1 VERSION="$(python -m setuptools_scm)" -for t in client exporter coordinator; do +for t in client exporter coordinator sftp-server; do ${DOCKER} build --build-arg VERSION="$VERSION" \ --target labgrid-${t} -t labgrid-${t} -f "${SCRIPT_DIR}/Dockerfile" . done diff --git a/labgrid/util/sshfs.py b/labgrid/util/sshfs.py new file mode 100644 index 000000000..7892feb2d --- /dev/null +++ b/labgrid/util/sshfs.py @@ -0,0 +1,152 @@ +import logging +import os +import subprocess +import random +import string +import time +import shlex +from contextlib import contextmanager + +import attr + +from .ssh import sshmanager +from .timeout import Timeout +from ..driver.exception import ExecutionError +from ..resource.common import Resource, NetworkResource + + +DEFAULT_SFTP_SERVER = "podman run --rm -i --mount type=bind,source={path},target={path} labgrid/sftp-server:latest usr/lib/ssh/sftp-server -e {readonly}" +DEFAULT_RO_OPT = "-R" + + +@attr.s +class SSHFsExport: + """ + Exports a local directory to a remote device using "reverse" SSH FS + """ + + local_path = attr.ib( + validator=attr.validators.instance_of(str), + converter=lambda x: os.path.abspath(str(x)), + ) + resource = attr.ib( + validator=attr.validators.instance_of(Resource), + ) + readonly = attr.ib( + default=True, + validator=attr.validators.instance_of(bool), + ) + + def __attrs_post_init__(self): + if not os.path.isdir(self.local_path): + raise FileNotFoundError(f"Local directory {self.local_path} not found") + self.logger = logging.getLogger(f"{self}") + + @contextmanager + def export(self, remote_path=None): + host = self.resource.host + conn = sshmanager.open(host) + + # If no remote path was specified, mount on a temporary directory + if remote_path is None: + tmpname = "".join(random.choices(string.ascii_lowercase, k=10)) + remote_path = f"/tmp/labgrid-sshfs/{tmpname}/" + conn.run_check(f"mkdir -p {remote_path}") + + env = self.resource.target.env + + if env is None: + sftp_server_opt = DEFAULT_SFTP_SERVER + ro_opt = DEFAULT_RO_OPT + else: + sftp_server_opt = env.config.get_option("sftp_server", DEFAULT_SFTP_SERVER) + ro_opt = env.config.get_option("sftp_server_readonly_opt", DEFAULT_RO_OPT) + + sftp_command = shlex.split( + sftp_server_opt.format( + path=self.local_path, + uid=str(os.getuid()), + gid=str(os.getgid()), + readonly=ro_opt if self.readonly else "", + ) + ) + + sshfs_command = conn.get_prefix() + [ + "sshfs", + "-o", + "slave", + "-o", + "idmap=user", + f":{self.local_path}", + remote_path, + ] + + self.logger.info( + "Running %s <-> %s", " ".join(sftp_command), " ".join(sshfs_command) + ) + + # Reverse sshfs requires that sftp-server running locally and sshfs + # running remotely each have their stdout connected to the others + # stdin. Connecting sftp-server stdout to sshfs stdin is done using + # Popen pipes, but in order to connect sshfs stdout to sftp-stdin, an + # external pipe is needed. + (rfd, wfd) = os.pipe2(os.O_CLOEXEC) + try: + with subprocess.Popen( + sftp_command, + stdout=subprocess.PIPE, + stdin=rfd, + ) as sftp_server, subprocess.Popen( + sshfs_command, + stdout=wfd, + stdin=sftp_server.stdout, + ) as sshfs: + # Close all file descriptor open in this process. This way, if + # either process exits, the other will get the EPIPE error and + # exit also. If this process doesn't close its copy of the + # descriptors, that won't happen + sftp_server.stdout.close() + + os.close(rfd) + rfd = -1 + + os.close(wfd) + wfd = -1 + + # Wait until the mount point appears remotely + t = Timeout(30.0) + + while not t.expired: + (_, _, exitcode) = conn.run(f"mountpoint --quiet {remote_path}") + + if exitcode == 0: + break + + if sshfs.poll() is not None: + raise ExecutionError( + "sshfs process exited with {sshfs.returncode}" + ) + + if sftp_server.poll() is not None: + raise ExecutionError( + "sftp process exited with {sftp_server.returncode}" + ) + + time.sleep(1) + + if t.expired: + raise TimeoutError("Timeout waiting for SSH fs to mount") + + try: + yield remote_path + finally: + sshfs.terminate() + sftp_server.terminate() + + finally: + # Cleanup if not done already + if rfd >= 0: + os.close(rfd) + + if wfd >= 0: + os.close(wfd) diff --git a/tests/test_util.py b/tests/test_util.py index ab83a8a5c..d2591fdcb 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -15,6 +15,7 @@ from labgrid.util.ssh import ForwardError, SSHConnection, sshmanager from labgrid.util.proxy import proxymanager from labgrid.util.managedfile import ManagedFile +from labgrid.util.sshfs import SSHFsExport from labgrid.driver.exception import ExecutionError from labgrid.resource.serialport import NetworkSerialPort from labgrid.resource.common import Resource, NetworkResource @@ -352,6 +353,49 @@ def test_local_managedfile(target, tmpdir): assert hash == mf.get_hash() assert str(t) == mf.get_remote_path() +@pytest.mark.localsshmanager +def test_sshfs(target, tmpdir): + t = tmpdir.join("test") + t_data = """ +Test +""" + t.write(t_data) + + res = NetworkResource(target, "test", "localhost") + sshfs = SSHFsExport(tmpdir, res) + + with sshfs.export() as remote_path: + remote_test = f"{remote_path}/test" + #assert os.path.exists(remote_test) + with open(remote_test, "r") as f: + remote_data = f.read() + + assert remote_data == t_data + + # Read-only by default + with pytest.raises(PermissionError): + with open(remote_test, "w") as f: + pass + +@pytest.mark.localsshmanager +def test_sshfs_write(target, tmpdir): + t = tmpdir.join("test") + t.write( +""" +Test +""" + ) + + res = NetworkResource(target, "test", "localhost") + sshfs = SSHFsExport(tmpdir, res, readonly=False) + + with sshfs.export() as remote_path: + remote_test = f"{remote_path}test" + #assert os.path.exists(remote_test) + with open(remote_test, "w") as f: + f.write("Hello") + + assert t.read() == "Hello" def test_find_dict(): dict_a = {"a": {"a.a": {"a.a.a": "a.a.a_val"}}, "b": "b_val"}