speed up prune/list/repo-list tests, see #9324

src/borg/conftest.py

@@ -66,6 +66,18 @@ def set_env_variables():
     os.environ["BORG_SELFTEST"] = "disabled"
 
 
+@pytest.fixture(scope="session")
+def backup_files(tmp_path_factory):
+    # create a relatively simple / minimal set of test files
+    path = tmp_path_factory.mktemp("backup")
+    (path / "empty").write_bytes(b"")
+    (path / "dir1").mkdir()
+    (path / "dir1" / "text.txt").write_text("text content")
+    (path / "dir2").mkdir()
+    (path / "dir2" / "binary.bin").write_bytes(b"\x00\x01\x02\x03")
+    return str(path)
+
+
 class ArchiverSetup:
     EXE: str = None  # python source based
     FORK_DEFAULT = False

src/borg/testsuite/archiver/__init__.py

@@ -36,6 +36,8 @@
 KF_ENCRYPTION = "--encryption=keyfile-chacha20-poly1305"
 
 # This points to the ``src/borg/archiver`` directory (small, with only a few files).
+# There are quite a lot of files in there, because there is a __pycache__ subdirectory.
+# Consider rather using the backup_files fixtures, which only has a few small files/dirs.
 src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "archiver"))
 src_file = "archiver/__init__.py"  # relative path of one file in src_dir
 

src/borg/testsuite/archiver/list_cmd_test.py

@@ -3,15 +3,15 @@
 import pytest
 
 from ...constants import *  # NOQA
-from . import src_dir, cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION, requires_hardlinks
+from . import cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION, requires_hardlinks
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary")  # NOQA
 
 
-def test_list_format(archivers, request):
+def test_list_format(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "test", src_dir)
+    cmd(archiver, "create", "test", backup_files)
     output_1 = cmd(archiver, "list", "test")
     output_2 = cmd(
         archiver, "list", "test", "--format", "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}"

src/borg/testsuite/archiver/prune_cmd_test.py

@@ -5,28 +5,28 @@
 
 from ...constants import *  # NOQA
 from ...archiver.prune_cmd import prune_split, prune_within
-from . import cmd, RK_ENCRYPTION, src_dir, generate_archiver_tests
+from . import cmd, RK_ENCRYPTION, generate_archiver_tests
 from ...helpers import interval
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary")  # NOQA
 
 
-def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0):
+def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0):
     cmd(
         archiver,
         "create",
         "--timestamp",
         datetime(y, m, d, H, M, S, 0).strftime(ISO_FORMAT_NO_USECS),  # naive == local time / local tz
         name,
-        src_dir,
+        backup_files,
     )
 
 
-def test_prune_repository(archivers, request):
+def test_prune_repository(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "test1", src_dir)
-    cmd(archiver, "create", "test2", src_dir)
+    cmd(archiver, "create", "test1", backup_files)
+    cmd(archiver, "create", "test2", backup_files)
     output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1")
     assert re.search(r"Would prune:\s+test1", output)
     # Must keep the latest archive:
@@ -42,41 +42,41 @@ def test_prune_repository(archivers, request):
 
 
 # This test must match docs/misc/prune-example.txt
-def test_prune_repository_example(archivers, request):
+def test_prune_repository_example(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
     # Archives that will be kept, per the example
     # Oldest archive
-    _create_archive_ts(archiver, "test01", 2015, 1, 1)
+    _create_archive_ts(archiver, backup_files, "test01", 2015, 1, 1)
     # 6 monthly archives
-    _create_archive_ts(archiver, "test02", 2015, 6, 30)
-    _create_archive_ts(archiver, "test03", 2015, 7, 31)
-    _create_archive_ts(archiver, "test04", 2015, 8, 31)
-    _create_archive_ts(archiver, "test05", 2015, 9, 30)
-    _create_archive_ts(archiver, "test06", 2015, 10, 31)
-    _create_archive_ts(archiver, "test07", 2015, 11, 30)
+    _create_archive_ts(archiver, backup_files, "test02", 2015, 6, 30)
+    _create_archive_ts(archiver, backup_files, "test03", 2015, 7, 31)
+    _create_archive_ts(archiver, backup_files, "test04", 2015, 8, 31)
+    _create_archive_ts(archiver, backup_files, "test05", 2015, 9, 30)
+    _create_archive_ts(archiver, backup_files, "test06", 2015, 10, 31)
+    _create_archive_ts(archiver, backup_files, "test07", 2015, 11, 30)
     # 14 daily archives
-    _create_archive_ts(archiver, "test08", 2015, 12, 17)
-    _create_archive_ts(archiver, "test09", 2015, 12, 18)
-    _create_archive_ts(archiver, "test10", 2015, 12, 20)
-    _create_archive_ts(archiver, "test11", 2015, 12, 21)
-    _create_archive_ts(archiver, "test12", 2015, 12, 22)
-    _create_archive_ts(archiver, "test13", 2015, 12, 23)
-    _create_archive_ts(archiver, "test14", 2015, 12, 24)
-    _create_archive_ts(archiver, "test15", 2015, 12, 25)
-    _create_archive_ts(archiver, "test16", 2015, 12, 26)
-    _create_archive_ts(archiver, "test17", 2015, 12, 27)
-    _create_archive_ts(archiver, "test18", 2015, 12, 28)
-    _create_archive_ts(archiver, "test19", 2015, 12, 29)
-    _create_archive_ts(archiver, "test20", 2015, 12, 30)
-    _create_archive_ts(archiver, "test21", 2015, 12, 31)
+    _create_archive_ts(archiver, backup_files, "test08", 2015, 12, 17)
+    _create_archive_ts(archiver, backup_files, "test09", 2015, 12, 18)
+    _create_archive_ts(archiver, backup_files, "test10", 2015, 12, 20)
+    _create_archive_ts(archiver, backup_files, "test11", 2015, 12, 21)
+    _create_archive_ts(archiver, backup_files, "test12", 2015, 12, 22)
+    _create_archive_ts(archiver, backup_files, "test13", 2015, 12, 23)
+    _create_archive_ts(archiver, backup_files, "test14", 2015, 12, 24)
+    _create_archive_ts(archiver, backup_files, "test15", 2015, 12, 25)
+    _create_archive_ts(archiver, backup_files, "test16", 2015, 12, 26)
+    _create_archive_ts(archiver, backup_files, "test17", 2015, 12, 27)
+    _create_archive_ts(archiver, backup_files, "test18", 2015, 12, 28)
+    _create_archive_ts(archiver, backup_files, "test19", 2015, 12, 29)
+    _create_archive_ts(archiver, backup_files, "test20", 2015, 12, 30)
+    _create_archive_ts(archiver, backup_files, "test21", 2015, 12, 31)
     # Additional archives that would be pruned
     # The second backup of the year
-    _create_archive_ts(archiver, "test22", 2015, 1, 2)
+    _create_archive_ts(archiver, backup_files, "test22", 2015, 1, 2)
     # The next older monthly backup
-    _create_archive_ts(archiver, "test23", 2015, 5, 31)
+    _create_archive_ts(archiver, backup_files, "test23", 2015, 5, 31)
     # The next older daily backup
-    _create_archive_ts(archiver, "test24", 2015, 12, 16)
+    _create_archive_ts(archiver, backup_files, "test24", 2015, 12, 16)
     output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1")
     # Prune second backup of the year
     assert re.search(r"Would prune:\s+test22", output)
@@ -104,7 +104,7 @@ def test_prune_repository_example(archivers, request):
         assert "test%02d" % i not in output
 
 
-def test_prune_quarterly(archivers, request):
+def test_prune_quarterly(archivers, request, backup_files):
     # Example worked through by hand when developing the quarterly
     # strategy, based on existing backups where the quarterly strategy
     # is desired. Weekly/monthly backups that do not affect results were
@@ -143,7 +143,7 @@ def mk_name(tup):
         # Initialize our repo.
         cmd(archiver, "repo-create", RK_ENCRYPTION)
         for a, (y, m, d) in zip(map(mk_name, test_dates), test_dates):
-            _create_archive_ts(archiver, a, y, m, d)
+            _create_archive_ts(archiver, backup_files, a, y, m, d)
 
         to_prune = list(set(test_dates) - set(to_keep))
 
@@ -176,38 +176,38 @@ def mk_name(tup):
 
 
 # With an initial and daily backup, prune daily until oldest is replaced by a monthly backup
-def test_prune_retain_and_expire_oldest(archivers, request):
+def test_prune_retain_and_expire_oldest(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
     # Initial backup
-    _create_archive_ts(archiver, "original_archive", 2020, 9, 1, 11, 15)
+    _create_archive_ts(archiver, backup_files, "original_archive", 2020, 9, 1, 11, 15)
     # Archive and prune daily for 30 days
     for i in range(1, 31):
-        _create_archive_ts(archiver, "september%02d" % i, 2020, 9, i, 12)
+        _create_archive_ts(archiver, backup_files, "september%02d" % i, 2020, 9, i, 12)
         cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1")
     # Archive and prune 6 days into the next month
     for i in range(1, 7):
-        _create_archive_ts(archiver, "october%02d" % i, 2020, 10, i, 12)
+        _create_archive_ts(archiver, backup_files, "october%02d" % i, 2020, 10, i, 12)
         cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1")
     # Oldest backup is still retained
     output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1")
     assert re.search(r"Keeping archive \(rule: monthly\[oldest\] #1" + r"\):\s+original_archive", output)
     # Archive one more day and prune.
-    _create_archive_ts(archiver, "october07", 2020, 10, 7, 12)
+    _create_archive_ts(archiver, backup_files, "october07", 2020, 10, 7, 12)
     cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1")
     # Last day of previous month is retained as monthly, and oldest is expired.
     output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1")
     assert re.search(r"Keeping archive \(rule: monthly #1\):\s+september30", output)
     assert "original_archive" not in output
 
 
-def test_prune_repository_prefix(archivers, request):
+def test_prune_repository_prefix(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "foo-2015-08-12-10:00", src_dir)
-    cmd(archiver, "create", "foo-2015-08-12-20:00", src_dir)
-    cmd(archiver, "create", "bar-2015-08-12-10:00", src_dir)
-    cmd(archiver, "create", "bar-2015-08-12-20:00", src_dir)
+    cmd(archiver, "create", "foo-2015-08-12-10:00", backup_files)
+    cmd(archiver, "create", "foo-2015-08-12-20:00", backup_files)
+    cmd(archiver, "create", "bar-2015-08-12-10:00", backup_files)
+    cmd(archiver, "create", "bar-2015-08-12-20:00", backup_files)
     output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:foo-*")
     assert re.search(r"Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00", output)
     assert re.search(r"Would prune:\s+foo-2015-08-12-10:00", output)
@@ -224,13 +224,13 @@ def test_prune_repository_prefix(archivers, request):
     assert "bar-2015-08-12-20:00" in output
 
 
-def test_prune_repository_glob(archivers, request):
+def test_prune_repository_glob(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "2015-08-12-10:00-foo", src_dir)
-    cmd(archiver, "create", "2015-08-12-20:00-foo", src_dir)
-    cmd(archiver, "create", "2015-08-12-10:00-bar", src_dir)
-    cmd(archiver, "create", "2015-08-12-20:00-bar", src_dir)
+    cmd(archiver, "create", "2015-08-12-10:00-foo", backup_files)
+    cmd(archiver, "create", "2015-08-12-20:00-foo", backup_files)
+    cmd(archiver, "create", "2015-08-12-10:00-bar", backup_files)
+    cmd(archiver, "create", "2015-08-12-20:00-bar", backup_files)
     output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:2015-*-foo")
     assert re.search(r"Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo", output)
     assert re.search(r"Would prune:\s+2015-08-12-10:00-foo", output)
@@ -402,15 +402,15 @@ def test_prune_split_no_archives():
     assert kept_because == {}
 
 
-def test_prune_list_with_metadata_format(archivers, request):
+def test_prune_list_with_metadata_format(archivers, request, backup_files):
     # Regression test for: prune --list with a format string that requires loading
     # archive metadata (e.g. {hostname}) must not fail when archives are deleted.
     # The bug was that format_item() was called after archive.delete(), causing
     # Archive.DoesNotExist when the formatter tried to lazy-load the archive.
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "test1", src_dir)
-    cmd(archiver, "create", "test2", src_dir)
+    cmd(archiver, "create", "test1", backup_files)
+    cmd(archiver, "create", "test2", backup_files)
     # {hostname} is a "call key" that triggers lazy loading of the archive from the repo.
     # With the buggy code this would raise Archive.DoesNotExist for the pruned archive.
     output = cmd(archiver, "prune", "--list", "--keep-daily=1", "--format={name} {hostname}{NL}")

src/borg/testsuite/archiver/repo_list_cmd_test.py

@@ -2,28 +2,29 @@
 import os
 
 from ...constants import *  # NOQA
-from . import cmd, checkts, create_src_archive, create_regular_file, src_dir, generate_archiver_tests, RK_ENCRYPTION
+from . import cmd, checkts, create_regular_file, generate_archiver_tests, RK_ENCRYPTION
+from .prune_cmd_test import _create_archive_ts
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary")  # NOQA
 
 
-def test_repo_list_glob(archivers, request):
+def test_repo_list_glob(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "test-1", src_dir)
-    cmd(archiver, "create", "something-else-than-test-1", src_dir)
-    cmd(archiver, "create", "test-2", src_dir)
+    cmd(archiver, "create", "test-1", backup_files)
+    cmd(archiver, "create", "something-else-than-test-1", backup_files)
+    cmd(archiver, "create", "test-2", backup_files)
     output = cmd(archiver, "repo-list", "--match-archives=sh:test-*")
     assert "test-1" in output
     assert "test-2" in output
     assert "something-else" not in output
 
 
-def test_archives_format(archivers, request):
+def test_archives_format(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "--comment", "comment 1", "test-1", src_dir)
-    cmd(archiver, "create", "--comment", "comment 2", "test-2", src_dir)
+    cmd(archiver, "create", "--comment", "comment 1", "test-1", backup_files)
+    cmd(archiver, "create", "--comment", "comment 2", "test-2", backup_files)
     output_1 = cmd(archiver, "repo-list")
     output_2 = cmd(
         archiver,
@@ -54,13 +55,13 @@ def test_size_nfiles(archivers, request):
     assert 123456 <= int(o_t[2]) < 123999  # There is some metadata overhead
 
 
-def test_date_matching(archivers, request):
+def test_date_matching(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
 
-    create_src_archive(archiver, "archive-2022-11-20", ts="2022-11-20T23:59:59")
-    create_src_archive(archiver, "archive-2022-12-18", ts="2022-12-18T23:59:59")
-    create_src_archive(archiver, "archive-now")
+    _create_archive_ts(archiver, backup_files, "archive-2022-11-20", 2022, 11, 20, 23, 59, 59)
+    _create_archive_ts(archiver, backup_files, "archive-2022-12-18", 2022, 12, 18, 23, 59, 59)
+    cmd(archiver, "create", "archive-now", backup_files)
 
     cmd(archiver, "check", "-v", "--oldest=23e", exit_code=2)
 
@@ -150,13 +151,13 @@ def test_repo_list_json(archivers, request):
     checkts(archive0["time"])
 
 
-def test_repo_list_deleted(archivers, request):
+def test_repo_list_deleted(archivers, request, backup_files):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    cmd(archiver, "create", "normal1", src_dir)
-    cmd(archiver, "create", "deleted1", src_dir)
-    cmd(archiver, "create", "normal2", src_dir)
-    cmd(archiver, "create", "deleted2", src_dir)
+    cmd(archiver, "create", "normal1", backup_files)
+    cmd(archiver, "create", "deleted1", backup_files)
+    cmd(archiver, "create", "normal2", backup_files)
+    cmd(archiver, "create", "deleted2", backup_files)
     cmd(archiver, "delete", "-a", "sh:deleted*")
     output = cmd(archiver, "repo-list")
     assert "normal1" in output