From 9177dc7c9ffd99bd009198ebc14dfa9361b02a4e Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Sat, 16 Jan 2021 15:07:17 -0800 Subject: [PATCH] unarchive: add support for .tar.zst (zstd compression) --- .../fragments/unarchive-support-zst.yml | 2 + lib/ansible/modules/unarchive.py | 25 +++++++++--- .../targets/unarchive/tasks/main.yml | 1 + .../targets/unarchive/tasks/prepare_tests.yml | 29 ++++++++++++++ .../targets/unarchive/tasks/test_tar_zst.yml | 40 +++++++++++++++++++ 5 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 changelogs/fragments/unarchive-support-zst.yml create mode 100644 test/integration/targets/unarchive/tasks/test_tar_zst.yml diff --git a/changelogs/fragments/unarchive-support-zst.yml b/changelogs/fragments/unarchive-support-zst.yml new file mode 100644 index 00000000000000..879efb7c4457a4 --- /dev/null +++ b/changelogs/fragments/unarchive-support-zst.yml @@ -0,0 +1,2 @@ +minor_changes: + - Add support in the unarchive module for .tar.zst (zstd compression) diff --git a/lib/ansible/modules/unarchive.py b/lib/ansible/modules/unarchive.py index 90d98f0a2e4ff5..d3737c82ae3d8d 100644 --- a/lib/ansible/modules/unarchive.py +++ b/lib/ansible/modules/unarchive.py @@ -110,8 +110,9 @@ - Re-implement zip support using native zipfile module. notes: - Requires C(zipinfo) and C(gtar)/C(unzip) command on target host. - - Can handle I(.zip) files using C(unzip) as well as I(.tar), I(.tar.gz), I(.tar.bz2) and I(.tar.xz) files using C(gtar). - - Does not handle I(.gz) files, I(.bz2) files or I(.xz) files that do not contain a I(.tar) archive. + - Requires C(zstd) command on target host to expand I(.tar.zst) files. + - Can handle I(.zip) files using C(unzip) as well as I(.tar), I(.tar.gz), I(.tar.bz2), I(.tar.xz), and I(.tar.zst) files using C(gtar). + - Does not handle I(.gz) files, I(.bz2) files, I(.xz), or I(.zst) files that do not contain a I(.tar) archive. - Uses gtar's C(--diff) arg to calculate if changed or not. If this C(arg) is not supported, it will always unpack the archive. - Existing files/directories in the destination which are not in the archive @@ -743,7 +744,7 @@ def files_in_archive(self): cmd = [self.cmd_path, '--list', '-C', self.b_dest] if self.zipflag: - cmd.append(self.zipflag) + cmd.extend(self.zipflag.split(' ')) if self.opts: cmd.extend(['--show-transformed-names'] + self.opts) if self.excludes: @@ -782,7 +783,7 @@ def files_in_archive(self): def is_unarchived(self): cmd = [self.cmd_path, '--diff', '-C', self.b_dest] if self.zipflag: - cmd.append(self.zipflag) + cmd.extend(self.zipflag.split(' ')) if self.opts: cmd.extend(['--show-transformed-names'] + self.opts) if self.file_args['owner']: @@ -835,7 +836,7 @@ def is_unarchived(self): def unarchive(self): cmd = [self.cmd_path, '--extract', '-C', self.b_dest] if self.zipflag: - cmd.append(self.zipflag) + cmd.extend(self.zipflag.split(' ')) if self.opts: cmd.extend(['--show-transformed-names'] + self.opts) if self.file_args['owner']: @@ -891,9 +892,21 @@ def __init__(self, src, b_dest, file_args, module): self.zipflag = '-J' +# Class to handle zstd compressed tar files +class TarZstdArchive(TgzArchive): + def __init__(self, src, b_dest, file_args, module): + super(TarZstdArchive, self).__init__(src, b_dest, file_args, module) + # GNU Tar supports the -I option to specify which executable + # to use for compression/decompression. + # + # Note: some flavors of BSD tar support --zstd (e.g., FreeBSD + # 12.2), but the TgzArchive class only supports GNU Tar. + self.zipflag = '-I zstd' + + # try handlers in order and return the one that works or bail if none work def pick_handler(src, dest, file_args, module): - handlers = [ZipArchive, TgzArchive, TarArchive, TarBzipArchive, TarXzArchive] + handlers = [ZipArchive, TgzArchive, TarArchive, TarBzipArchive, TarXzArchive, TarZstdArchive] reasons = set() for handler in handlers: obj = handler(src, dest, file_args, module) diff --git a/test/integration/targets/unarchive/tasks/main.yml b/test/integration/targets/unarchive/tasks/main.yml index a6acea6e36a94b..52b4bff492663a 100644 --- a/test/integration/targets/unarchive/tasks/main.yml +++ b/test/integration/targets/unarchive/tasks/main.yml @@ -4,6 +4,7 @@ - import_tasks: test_tar_gz_creates.yml - import_tasks: test_tar_gz_owner_group.yml - import_tasks: test_tar_gz_keep_newer.yml +- import_tasks: test_tar_zst.yml - import_tasks: test_zip.yml - import_tasks: test_exclude.yml - import_tasks: test_include.yml diff --git a/test/integration/targets/unarchive/tasks/prepare_tests.yml b/test/integration/targets/unarchive/tasks/prepare_tests.yml index 4025b0f2dcf7ca..5d652c6a0ca8d2 100644 --- a/test/integration/targets/unarchive/tasks/prepare_tests.yml +++ b/test/integration/targets/unarchive/tasks/prepare_tests.yml @@ -6,6 +6,13 @@ - unzip when: ansible_pkg_mgr in ('yum', 'dnf', 'apt', 'pkgng') +- name: Ensure zstd is present, if available + ignore_errors: true + package: + name: + - zstd + when: ansible_pkg_mgr in ('yum', 'dnf', 'apt', 'pkgng') + - name: prep our file copy: src: foo.txt @@ -18,6 +25,28 @@ - name: prep a tar.gz file shell: tar czvf test-unarchive.tar.gz foo-unarchive.txt chdir={{remote_tmp_dir}} +- name: see if we have the zstd executable + ignore_errors: true + shell: zstd --version + register: zstd_available + +- when: zstd_available.rc == 0 + block: + - name: find gnu tar + shell: | + #!/bin/sh + which gtar 2>/dev/null + if test $? -ne 0; then + if test -z "`tar --version | grep bsdtar`"; then + which tar + fi + fi + register: gnu_tar + + - name: prep a tar.zst file + shell: "{{ gnu_tar.stdout }} -I zstd -cvf test-unarchive.tar.zst foo-unarchive.txt chdir={{remote_tmp_dir}}" + when: gnu_tar.stdout != "" + - name: prep a chmodded file for zip copy: src: foo.txt diff --git a/test/integration/targets/unarchive/tasks/test_tar_zst.yml b/test/integration/targets/unarchive/tasks/test_tar_zst.yml new file mode 100644 index 00000000000000..18b128159f5310 --- /dev/null +++ b/test/integration/targets/unarchive/tasks/test_tar_zst.yml @@ -0,0 +1,40 @@ +# Only do this whole file when the "zstd" executable is present +- when: + - zstd_available.rc == 0 + - gnu_tar.stdout != "" + block: + - name: create our tar.zst unarchive destination + file: + path: '{{remote_tmp_dir}}/test-unarchive-tar-zst' + state: directory + + - name: unarchive a tar.zst file + unarchive: + src: '{{remote_tmp_dir}}/test-unarchive.tar.zst' + dest: '{{remote_tmp_dir}}/test-unarchive-tar-zst' + remote_src: yes + register: unarchive02 + + - name: verify that the file was marked as changed + assert: + that: + - "unarchive02.changed == true" + # Verify that no file list is generated + - "'files' not in unarchive02" + + - name: verify that the file was unarchived + file: + path: '{{remote_tmp_dir}}/test-unarchive-tar-zst/foo-unarchive.txt' + state: file + + - name: remove our tar.zst unarchive destination + file: + path: '{{remote_tmp_dir}}/test-unarchive-tar-zst' + state: absent + + - name: test owner/group perms + include_tasks: test_owner_group.yml + vars: + ext: tar.zst + archive: test-unarchive.tar.zst + testfile: foo-unarchive.txt