Enable more common compression filters for restoring

Since some users want pigz, zstd and some other compressions which might
not be installed by default, enable them if they are installed.

Related: QubesOS/qubes-issues#8291
Related: QubesOS/qubes-issues#8211

Author: alimirjamali

.gitlab-ci.yml

@@ -20,6 +20,7 @@ checks:tests:
   - ci/codecov-wrapper
   before_script:
   - sudo dnf install -y openssl python3-rpm sequoia-sqv python3-xlib libnotify
+  - sudo dnf install -y zstd
   - sudo dnf install -y --enablerepo=qubes-host-r4.3-current-testing scrypt
   - pip3 install --quiet -r ci/requirements.txt
   - git config --global --add safe.directory "$PWD"

qubesadmin/backup/restore.py

@@ -66,6 +66,7 @@
 DEFAULT_HMAC_ALGORITHM = 'scrypt'
 DEFAULT_COMPRESSION_FILTER = 'gzip'
 KNOWN_COMPRESSION_FILTERS = ('gzip', 'bzip2', 'xz')
+OPTIONAL_COMPRESSION_FILTERS = ('lzma', 'pigz', 'zstd', 'zstdmt')
 # lazy loaded
 KNOWN_CRYPTO_ALGORITHMS = []
 # lazy loaded
@@ -101,6 +102,13 @@ def init_supported_hmac_and_crypto():
     if not KNOWN_CRYPTO_ALGORITHMS:
         KNOWN_CRYPTO_ALGORITHMS.extend(get_supported_crypto_algo())
 
+def validate_compression_filter(compressor: str):
+    '''Check if the compression algorithm is available on the system'''
+    if compressor in KNOWN_COMPRESSION_FILTERS:
+        return True
+    if compressor in OPTIONAL_COMPRESSION_FILTERS and shutil.which(compressor):
+        return True
+    return False
 
 class BackupHeader(object):
     '''Structure describing backup-header file included as the first file in
@@ -118,7 +126,7 @@ class BackupHeader(object):
         'compression-filter': Header(
             field='compression_filter',
             t=str,
-            validator=lambda x: x in KNOWN_COMPRESSION_FILTERS),
+            validator=validate_compression_filter),
         'crypto-algorithm': Header(
             field='crypto_algorithm',
             t=str,
@@ -203,6 +211,11 @@ def load(self, untrusted_header_text):
                 raise QubesException("Unrecognized header type")
             if not header.validator(value):
                 if key == 'compression-filter':
+                    if value in OPTIONAL_COMPRESSION_FILTERS:
+                        raise QubesException(
+                            f"Optional '{value}' compression filter is not "
+                            "installed. Install it and retry."
+                        )
                     raise QubesException(
                         "Unusual compression filter '{f}' found. Use "
                         "--compression-filter={f} to use it anyway.".format(

qubesadmin/tests/backup/backupcompatibility.py

@@ -2046,6 +2046,83 @@ def test_230_r4_uncommon_cmpression_forced(self):
 
         self.assertDom0Restored(dummy_timestamp)
 
+    @unittest.skipUnless(shutil.which('scrypt'),
+        "scrypt not installed")
+    def test_230_r4_optional_compression(self):
+        self.create_v4_backup("zstdmt")
+
+        self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
+            b'0\0dom0 class=AdminVM state=Running\n'
+            b'fedora-25 class=TemplateVM state=Halted\n'
+            b'testvm class=AppVM state=Running\n'
+            b'sys-net class=AppVM state=Running\n'
+        )
+        self.app.expected_calls[
+            ('dom0', 'admin.property.Get', 'default_template', None)] = \
+            b'0\0default=no type=vm fedora-25'
+        self.app.expected_calls[
+            ('sys-net', 'admin.vm.property.Get', 'provides_network', None)] = \
+            b'0\0default=no type=bool True'
+        self.setup_expected_calls(parsed_qubes_xml_v4, templates_map={
+            'debian-8': 'fedora-25'
+        })
+        firewall_data = (
+            'action=accept specialtarget=dns\n'
+            'action=accept proto=icmp\n'
+            'action=accept proto=tcp dstports=22-22\n'
+            'action=accept proto=tcp dsthost=www.qubes-os.org '
+            'dstports=443-443\n'
+            'action=accept proto=tcp dst4=192.168.0.0/24\n'
+            'action=drop\n'
+        )
+        self.app.expected_calls[
+            ('test-work', 'admin.vm.firewall.Set', None,
+            firewall_data.encode())] = b'0\0'
+
+        qubesd_calls_queue = multiprocessing.Queue()
+
+        dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
+        patches = [
+            mock.patch('qubesadmin.storage.Volume',
+                staticmethod(
+                    functools.partial(MockVolume, qubesd_calls_queue, 0)),
+                ),
+            mock.patch(
+                'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
+                staticmethod(
+                    functools.partial(self.mock_appmenus, qubesd_calls_queue)),
+                ),
+            mock.patch(
+                'qubesadmin.firewall.Firewall',
+                staticmethod(
+                    functools.partial(MockFirewall, qubesd_calls_queue)),
+                ),
+            mock.patch(
+                'time.strftime',
+                return_value=dummy_timestamp)
+        ]
+        for patch in patches:
+            patch.start()
+        try:
+            self.restore_backup(self.fullpath("backup.bin"), options={
+                'use-default-template': True,
+                'use-default-netvm': True,
+            })
+        finally:
+            for patch in patches:
+                patch.stop()
+
+        # retrieve calls from other multiprocess.Process instances
+        while not qubesd_calls_queue.empty():
+            call_args = qubesd_calls_queue.get()
+            with contextlib.suppress(qubesadmin.exc.QubesException):
+                self.app.qubesd_call(*call_args)
+        qubesd_calls_queue.close()
+
+        self.assertAllCalled()
+
+        self.assertDom0Restored(dummy_timestamp)
+
     @unittest.skipUnless(shutil.which('scrypt'),
         "scrypt not installed")
     def test_300_r4_no_space(self):