From 1d1172e742732655f8cb13321b64a2c9ad9e3a18 Mon Sep 17 00:00:00 2001 From: Jordan Webb Date: Fri, 7 Jun 2019 12:38:29 -0500 Subject: [PATCH] Stop using `buildah mount` Instead, use `buildah copy` for `put_file` and `buildah run dd` for `fetch_file` Also, fix pipelining Borrows a bunch of code from chroot.py --- lib/ansible/plugins/connection/buildah.py | 94 +++++++++++++++-------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/lib/ansible/plugins/connection/buildah.py b/lib/ansible/plugins/connection/buildah.py index 470173be18f19f..22ad3298aa7848 100644 --- a/lib/ansible/plugins/connection/buildah.py +++ b/lib/ansible/plugins/connection/buildah.py @@ -41,14 +41,17 @@ # - name: remote_user """ +import os import shlex import shutil - import subprocess +import traceback import ansible.constants as C +from ansible.errors import AnsibleError from ansible.module_utils._text import to_bytes, to_native -from ansible.plugins.connection import ConnectionBase, ensure_connect +from ansible.module_utils.six.moves import shlex_quote +from ansible.plugins.connection import ConnectionBase, BUFSIZE from ansible.utils.display import Display display = Display() @@ -78,14 +81,14 @@ def __init__(self, play_context, new_stdin, *args, **kwargs): def _set_user(self): self._buildah(b"config", [b"--user=" + to_bytes(self.user, errors='surrogate_or_strict')]) - def _buildah(self, cmd, cmd_args=None, in_data=None): + def _buffered_buildah(self, cmd, cmd_args=None, stdin=subprocess.PIPE): """ run buildah executable :param cmd: buildah's command to execute (str) :param cmd_args: list of arguments to pass to the command (list of str/bytes) - :param in_data: data passed to buildah's stdin - :return: return code, stdout, stderr + :param stdin: passed to subproces.Popen + :return: subprocess.Popen object """ local_cmd = ['buildah', cmd, '--', self._container_id] if cmd_args: @@ -93,9 +96,22 @@ def _buildah(self, cmd, cmd_args=None, in_data=None): local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] display.vvv("RUN %s" % (local_cmd,), host=self._container_id) - p = subprocess.Popen(local_cmd, shell=False, stdin=subprocess.PIPE, + p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return p + + def _buildah(self, cmd, cmd_args=None, in_data=None): + """ + run buildah executable + + :param cmd: buildah's command to execute (str) + :param cmd_args: list of arguments to pass to the command (list of str/bytes) + :param in_data: data passed to buildah's stdin + :return: return code, stdout, stderr + """ + p = self._buffered_buildah(cmd, cmd_args) + stdout, stderr = p.communicate(input=in_data) stdout = to_bytes(stdout, errors='surrogate_or_strict') stderr = to_bytes(stderr, errors='surrogate_or_strict') @@ -103,16 +119,13 @@ def _buildah(self, cmd, cmd_args=None, in_data=None): def _connect(self): """ - no persistent connection is being maintained, mount container's filesystem - so we can easily access it + no persistent connection is being maintained """ super(Connection, self)._connect() - rc, self._mount_point, stderr = self._buildah("mount") - self._mount_point = self._mount_point.strip() - display.vvvvv("MOUNTPOINT %s RC %s STDERR %r" % (self._mount_point, rc, stderr)) - self._connected = True + if not self._connected: + display.vvv("THIS IS A BUILDAH CONTAINER", host=self._container_id) + self._connected = True - @ensure_connect def exec_command(self, cmd, in_data=None, sudoable=False): """ run specified command in a running OCI container using buildah """ super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) @@ -120,42 +133,61 @@ def exec_command(self, cmd, in_data=None, sudoable=False): # shlex.split has a bug with text strings on Python-2.6 and can only handle text strings on Python-3 cmd_args_list = shlex.split(to_native(cmd, errors='surrogate_or_strict')) - rc, stdout, stderr = self._buildah("run", cmd_args_list) + rc, stdout, stderr = self._buildah("run", cmd_args_list, in_data) display.vvvvv("STDOUT %r STDERR %r" % (stderr, stderr)) return rc, stdout, stderr + def _prefix_login_path(self, remote_path): + ''' Make sure that we put files into a standard path + + If a path is relative, then we need to choose where to put it. + ssh chooses $HOME but we aren't guaranteed that a home dir will + exist in any given chroot. So for now we're choosing "/" instead. + This also happens to be the former default. + + Can revisit using $HOME instead if it's a problem + ''' + if not remote_path.startswith(os.path.sep): + remote_path = os.path.join(os.path.sep, remote_path) + return os.path.normpath(remote_path) + def put_file(self, in_path, out_path): """ Place a local file located in 'in_path' inside container at 'out_path' """ super(Connection, self).put_file(in_path, out_path) display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._container_id) - real_out_path = self._mount_point + to_bytes(out_path, errors='surrogate_or_strict') - shutil.copyfile( - to_bytes(in_path, errors='surrogate_or_strict'), - to_bytes(real_out_path, errors='surrogate_or_strict') + rc, stdout, stderr = self._buildah( + "copy", + [to_bytes(in_path, errors='surrogate_or_strict'), + to_bytes(self._prefix_login_path(out_path), errors='surrogate_or_strict')] ) - # alternatively, this can be implemented using `buildah copy`: - # rc, stdout, stderr = self._buildah( - # "copy", - # [to_bytes(in_path, errors='surrogate_or_strict'), - # to_bytes(out_path, errors='surrogate_or_strict')] - # ) + + if rc != 0: + raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr)) def fetch_file(self, in_path, out_path): """ obtain file specified via 'in_path' from the container and place it at 'out_path' """ super(Connection, self).fetch_file(in_path, out_path) display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._container_id) - real_in_path = self._mount_point + to_bytes(in_path, errors='surrogate_or_strict') - shutil.copyfile( - to_bytes(real_in_path, errors='surrogate_or_strict'), - to_bytes(out_path, errors='surrogate_or_strict') - ) + in_path = shlex_quote(self._prefix_login_path(in_path)) + p = self._buffered_buildah("run", ['dd', 'if=%s' % in_path, 'bs=%s' % BUFSIZE]) + + with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file: + try: + chunk = p.stdout.read(BUFSIZE) + while chunk: + out_file.write(chunk) + chunk = p.stdout.read(BUFSIZE) + except Exception: + traceback.print_exc() + raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path)) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr)) def close(self): """ unmount container's filesystem """ super(Connection, self).close() - rc, stdout, stderr = self._buildah("umount") - display.vvvvv("RC %s STDOUT %r STDERR %r" % (rc, stdout, stderr)) self._connected = False