CLI: Added compress option to verdi storage maintain (#5540)

By default, the disk-object store, which is used by the default storage
backend `PsqlDosStorage`, does _not_ compress files when joining them
into pack files during the maintenace operation. This behavior can be
toggled through a flag in the interface, but this is not exposed in the
`verdi storage maintain` and so AiiDA users currently cannot benefit
from file compression.

Moreover, in the current implementation of the disk-object store, once
packed with or without compression, it cannot be repacked with a
different setting.

Here the `--compress` option is added to `verdi storage maintain` which
can change the default behavior of not compressing when packing loose
files.

Note that this is a storage backend dependent option that not all storage
backends may support, but currently the spread of alternative storage
backends in the AiiDA ecosystem is very limited and it is unlikely that
a fully dynamic interface for `verdi storage maintain` is necessary in
the near future. If this need arises in the future, the `--compress`
option might have to be deprecated, but this is an acceptable cost to
prevent from having to implement a fully dynamic CLI interface at this
point in time.

aiida/cmdline/commands/cmd_storage.py

@@ -119,9 +119,12 @@ def storage_info(detailed):
     help=
     'Run the maintenance in dry-run mode which will print actions that would be taken without actually executing them.'
 )
+@click.option(
+    '--compress', is_flag=True, default=False, help='Use compression if possible when carrying out maintenance tasks.'
+)
 @decorators.with_dbenv()
 @click.pass_context
-def storage_maintain(ctx, full, no_repack, force, dry_run):
+def storage_maintain(ctx, full, no_repack, force, dry_run, compress):
     """Performs maintenance tasks on the repository."""
     from aiida.common.exceptions import LockingProfileError
     from aiida.manage.manager import get_manager
@@ -157,9 +160,9 @@ def storage_maintain(ctx, full, no_repack, force, dry_run):
 
     try:
         if full and no_repack:
-            storage.maintain(full=full, dry_run=dry_run, do_repack=False)
+            storage.maintain(full=full, dry_run=dry_run, do_repack=False, compress=compress)
         else:
-            storage.maintain(full=full, dry_run=dry_run)
+            storage.maintain(full=full, dry_run=dry_run, compress=compress)
     except LockingProfileError as exception:
         echo.echo_critical(str(exception))
     echo.echo_success('Requested maintenance procedures finished.')

aiida/repository/backend/disk_object_store.py

@@ -149,6 +149,7 @@ def maintain( # type: ignore[override] # pylint: disable=arguments-differ,too-ma
         do_repack: bool = None,
         clean_storage: bool = None,
         do_vacuum: bool = None,
+        compress: bool = False,
     ) -> dict:
         """Performs maintenance operations.
 
@@ -157,6 +158,7 @@ def maintain( # type: ignore[override] # pylint: disable=arguments-differ,too-ma
         :param do_repack:flag for forcing the re-packing of already packed files.
         :param clean_storage:flag for forcing the cleaning of soft-deleted files from the repository.
         :param do_vacuum:flag for forcing the vacuuming of the internal database when cleaning the repository.
+        :param compress:flag for compressing the data when packing loose files.
         :return:a dictionary with information on the operations performed.
         """
         if live and (do_repack or clean_storage or do_vacuum):
@@ -181,7 +183,7 @@ def maintain( # type: ignore[override] # pylint: disable=arguments-differ,too-ma
                 files_size = container.get_total_size()['total_size_loose'] * BYTES_TO_MB
                 logger.report(f'Packing all loose files ({files_numb} files occupying {files_size} MB) ...')
                 if not dry_run:
-                    container.pack_all_loose()
+                    container.pack_all_loose(compress=compress)
 
             if do_repack:
                 files_numb = container.count_objects()['packed']

tests/cmdline/commands/test_storage.py

@@ -143,16 +143,18 @@ def mock_maintain(*args, **kwargs):
     assert message_list[-1] == 'Are you sure you want continue in this mode? [y/N]: '
 
     # Test `storage.mantain` with `--force`
-    result = run_cli_command(cmd_storage.storage_maintain, parameters=['--force'], use_subprocess=False)
+    result = run_cli_command(cmd_storage.storage_maintain, parameters=['--force', '--compress'], use_subprocess=False)
     message_list = result.output_lines
     assert ' > full: False' in message_list
     assert ' > dry_run: False' in message_list
+    assert ' > compress: True' in message_list
 
     # Test `storage.mantain` with user input Y
     result = run_cli_command(cmd_storage.storage_maintain, user_input='Y', use_subprocess=False)
     message_list = result.output_lines
     assert ' > full: False' in message_list
     assert ' > dry_run: False' in message_list
+    assert ' > compress: False' in message_list
 
     # Test `storage.mantain` with `--dry-run`
     result = run_cli_command(cmd_storage.storage_maintain, parameters=['--dry-run'], use_subprocess=False)

tests/repository/backend/test_disk_object_store.py

@@ -249,6 +249,10 @@ def test_get_info(populated_repository):
         {'live': False},
         {'unpacked': 0, 'packed': 4}
     ),
+    (
+        {'live': False, 'compress': True},
+        {'unpacked': 0, 'packed': 4}
+    ),
     (
         {'live': False, 'do_vacuum': False},
         {'unpacked': 0, 'packed': 4}

tests/storage/psql_dos/test_backend.py

@@ -68,11 +68,15 @@ def test_get_unreferenced_keyset():
     ),
     ((
         {},
-        [' > live: True', ' > dry_run: False']
+        [' > live: True', ' > dry_run: False', ' > compress: False']
     ),
     (
         {'full': True, 'dry_run': True},
-        [' > live: False', ' > dry_run: True']
+        [' > live: False', ' > dry_run: True', ' > compress: False']
+    ),
+    (
+        {'full': True, 'dry_run': True, 'compress': True},
+        [' > live: False', ' > dry_run: True', ' > compress: True']
     ),
     (
         {'extra_kwarg': 'molly'},
@@ -87,10 +91,11 @@ def test_maintain(caplog, monkeypatch, kwargs, logged_texts):
 
     storage_backend = get_manager().get_profile_storage()
 
-    def mock_maintain(self, live=True, dry_run=False, **kwargs):  # pylint: disable=unused-argument
+    def mock_maintain(self, live=True, dry_run=False, compress=False, **kwargs):  # pylint: disable=unused-argument
         logmsg = 'keywords provided:\n'
         logmsg += f' > live: {live}\n'
         logmsg += f' > dry_run: {dry_run}\n'
+        logmsg += f' > compress: {compress}\n'
         for key, val in kwargs.items():
             logmsg += f' > {key}: {val}\n'
         logging.info(logmsg)