diff --git a/src/borg/conftest.py b/src/borg/conftest.py index ebe1be1807..57b2dcbdaa 100644 --- a/src/borg/conftest.py +++ b/src/borg/conftest.py @@ -66,6 +66,18 @@ def set_env_variables(): os.environ["BORG_SELFTEST"] = "disabled" +@pytest.fixture(scope="session") +def backup_files(tmp_path_factory): + # create a relatively simple / minimal set of test files + path = tmp_path_factory.mktemp("backup") + (path / "empty").write_bytes(b"") + (path / "dir1").mkdir() + (path / "dir1" / "text.txt").write_text("text content") + (path / "dir2").mkdir() + (path / "dir2" / "binary.bin").write_bytes(b"\x00\x01\x02\x03") + return str(path) + + class ArchiverSetup: EXE: str = None # python source based FORK_DEFAULT = False diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index 667bf413d7..ebb27f7904 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -36,6 +36,8 @@ KF_ENCRYPTION = "--encryption=keyfile-chacha20-poly1305" # This points to the ``src/borg/archiver`` directory (small, with only a few files). +# There are quite a lot of files in there, because there is a __pycache__ subdirectory. +# Consider rather using the backup_files fixtures, which only has a few small files/dirs. src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "archiver")) src_file = "archiver/__init__.py" # relative path of one file in src_dir diff --git a/src/borg/testsuite/archiver/list_cmd_test.py b/src/borg/testsuite/archiver/list_cmd_test.py index 20224b4ae5..f9bb56e58a 100644 --- a/src/borg/testsuite/archiver/list_cmd_test.py +++ b/src/borg/testsuite/archiver/list_cmd_test.py @@ -3,15 +3,15 @@ import pytest from ...constants import * # NOQA -from . import src_dir, cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION, requires_hardlinks +from . import cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION, requires_hardlinks pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA -def test_list_format(archivers, request): +def test_list_format(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "test", src_dir) + cmd(archiver, "create", "test", backup_files) output_1 = cmd(archiver, "list", "test") output_2 = cmd( archiver, "list", "test", "--format", "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}" diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index fff66905bf..abf431c1c1 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -5,28 +5,28 @@ from ...constants import * # NOQA from ...archiver.prune_cmd import prune_split, prune_within -from . import cmd, RK_ENCRYPTION, src_dir, generate_archiver_tests +from . import cmd, RK_ENCRYPTION, generate_archiver_tests from ...helpers import interval pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA -def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0): +def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0): cmd( archiver, "create", "--timestamp", datetime(y, m, d, H, M, S, 0).strftime(ISO_FORMAT_NO_USECS), # naive == local time / local tz name, - src_dir, + backup_files, ) -def test_prune_repository(archivers, request): +def test_prune_repository(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "test1", src_dir) - cmd(archiver, "create", "test2", src_dir) + cmd(archiver, "create", "test1", backup_files) + cmd(archiver, "create", "test2", backup_files) output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1") assert re.search(r"Would prune:\s+test1", output) # Must keep the latest archive: @@ -42,41 +42,41 @@ def test_prune_repository(archivers, request): # This test must match docs/misc/prune-example.txt -def test_prune_repository_example(archivers, request): +def test_prune_repository_example(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) # Archives that will be kept, per the example # Oldest archive - _create_archive_ts(archiver, "test01", 2015, 1, 1) + _create_archive_ts(archiver, backup_files, "test01", 2015, 1, 1) # 6 monthly archives - _create_archive_ts(archiver, "test02", 2015, 6, 30) - _create_archive_ts(archiver, "test03", 2015, 7, 31) - _create_archive_ts(archiver, "test04", 2015, 8, 31) - _create_archive_ts(archiver, "test05", 2015, 9, 30) - _create_archive_ts(archiver, "test06", 2015, 10, 31) - _create_archive_ts(archiver, "test07", 2015, 11, 30) + _create_archive_ts(archiver, backup_files, "test02", 2015, 6, 30) + _create_archive_ts(archiver, backup_files, "test03", 2015, 7, 31) + _create_archive_ts(archiver, backup_files, "test04", 2015, 8, 31) + _create_archive_ts(archiver, backup_files, "test05", 2015, 9, 30) + _create_archive_ts(archiver, backup_files, "test06", 2015, 10, 31) + _create_archive_ts(archiver, backup_files, "test07", 2015, 11, 30) # 14 daily archives - _create_archive_ts(archiver, "test08", 2015, 12, 17) - _create_archive_ts(archiver, "test09", 2015, 12, 18) - _create_archive_ts(archiver, "test10", 2015, 12, 20) - _create_archive_ts(archiver, "test11", 2015, 12, 21) - _create_archive_ts(archiver, "test12", 2015, 12, 22) - _create_archive_ts(archiver, "test13", 2015, 12, 23) - _create_archive_ts(archiver, "test14", 2015, 12, 24) - _create_archive_ts(archiver, "test15", 2015, 12, 25) - _create_archive_ts(archiver, "test16", 2015, 12, 26) - _create_archive_ts(archiver, "test17", 2015, 12, 27) - _create_archive_ts(archiver, "test18", 2015, 12, 28) - _create_archive_ts(archiver, "test19", 2015, 12, 29) - _create_archive_ts(archiver, "test20", 2015, 12, 30) - _create_archive_ts(archiver, "test21", 2015, 12, 31) + _create_archive_ts(archiver, backup_files, "test08", 2015, 12, 17) + _create_archive_ts(archiver, backup_files, "test09", 2015, 12, 18) + _create_archive_ts(archiver, backup_files, "test10", 2015, 12, 20) + _create_archive_ts(archiver, backup_files, "test11", 2015, 12, 21) + _create_archive_ts(archiver, backup_files, "test12", 2015, 12, 22) + _create_archive_ts(archiver, backup_files, "test13", 2015, 12, 23) + _create_archive_ts(archiver, backup_files, "test14", 2015, 12, 24) + _create_archive_ts(archiver, backup_files, "test15", 2015, 12, 25) + _create_archive_ts(archiver, backup_files, "test16", 2015, 12, 26) + _create_archive_ts(archiver, backup_files, "test17", 2015, 12, 27) + _create_archive_ts(archiver, backup_files, "test18", 2015, 12, 28) + _create_archive_ts(archiver, backup_files, "test19", 2015, 12, 29) + _create_archive_ts(archiver, backup_files, "test20", 2015, 12, 30) + _create_archive_ts(archiver, backup_files, "test21", 2015, 12, 31) # Additional archives that would be pruned # The second backup of the year - _create_archive_ts(archiver, "test22", 2015, 1, 2) + _create_archive_ts(archiver, backup_files, "test22", 2015, 1, 2) # The next older monthly backup - _create_archive_ts(archiver, "test23", 2015, 5, 31) + _create_archive_ts(archiver, backup_files, "test23", 2015, 5, 31) # The next older daily backup - _create_archive_ts(archiver, "test24", 2015, 12, 16) + _create_archive_ts(archiver, backup_files, "test24", 2015, 12, 16) output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") # Prune second backup of the year assert re.search(r"Would prune:\s+test22", output) @@ -104,7 +104,7 @@ def test_prune_repository_example(archivers, request): assert "test%02d" % i not in output -def test_prune_quarterly(archivers, request): +def test_prune_quarterly(archivers, request, backup_files): # Example worked through by hand when developing the quarterly # strategy, based on existing backups where the quarterly strategy # is desired. Weekly/monthly backups that do not affect results were @@ -143,7 +143,7 @@ def mk_name(tup): # Initialize our repo. cmd(archiver, "repo-create", RK_ENCRYPTION) for a, (y, m, d) in zip(map(mk_name, test_dates), test_dates): - _create_archive_ts(archiver, a, y, m, d) + _create_archive_ts(archiver, backup_files, a, y, m, d) to_prune = list(set(test_dates) - set(to_keep)) @@ -176,24 +176,24 @@ def mk_name(tup): # With an initial and daily backup, prune daily until oldest is replaced by a monthly backup -def test_prune_retain_and_expire_oldest(archivers, request): +def test_prune_retain_and_expire_oldest(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) # Initial backup - _create_archive_ts(archiver, "original_archive", 2020, 9, 1, 11, 15) + _create_archive_ts(archiver, backup_files, "original_archive", 2020, 9, 1, 11, 15) # Archive and prune daily for 30 days for i in range(1, 31): - _create_archive_ts(archiver, "september%02d" % i, 2020, 9, i, 12) + _create_archive_ts(archiver, backup_files, "september%02d" % i, 2020, 9, i, 12) cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") # Archive and prune 6 days into the next month for i in range(1, 7): - _create_archive_ts(archiver, "october%02d" % i, 2020, 10, i, 12) + _create_archive_ts(archiver, backup_files, "october%02d" % i, 2020, 10, i, 12) cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") # Oldest backup is still retained output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") assert re.search(r"Keeping archive \(rule: monthly\[oldest\] #1" + r"\):\s+original_archive", output) # Archive one more day and prune. - _create_archive_ts(archiver, "october07", 2020, 10, 7, 12) + _create_archive_ts(archiver, backup_files, "october07", 2020, 10, 7, 12) cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") # Last day of previous month is retained as monthly, and oldest is expired. output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") @@ -201,13 +201,13 @@ def test_prune_retain_and_expire_oldest(archivers, request): assert "original_archive" not in output -def test_prune_repository_prefix(archivers, request): +def test_prune_repository_prefix(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "foo-2015-08-12-10:00", src_dir) - cmd(archiver, "create", "foo-2015-08-12-20:00", src_dir) - cmd(archiver, "create", "bar-2015-08-12-10:00", src_dir) - cmd(archiver, "create", "bar-2015-08-12-20:00", src_dir) + cmd(archiver, "create", "foo-2015-08-12-10:00", backup_files) + cmd(archiver, "create", "foo-2015-08-12-20:00", backup_files) + cmd(archiver, "create", "bar-2015-08-12-10:00", backup_files) + cmd(archiver, "create", "bar-2015-08-12-20:00", backup_files) output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:foo-*") assert re.search(r"Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00", output) assert re.search(r"Would prune:\s+foo-2015-08-12-10:00", output) @@ -224,13 +224,13 @@ def test_prune_repository_prefix(archivers, request): assert "bar-2015-08-12-20:00" in output -def test_prune_repository_glob(archivers, request): +def test_prune_repository_glob(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "2015-08-12-10:00-foo", src_dir) - cmd(archiver, "create", "2015-08-12-20:00-foo", src_dir) - cmd(archiver, "create", "2015-08-12-10:00-bar", src_dir) - cmd(archiver, "create", "2015-08-12-20:00-bar", src_dir) + cmd(archiver, "create", "2015-08-12-10:00-foo", backup_files) + cmd(archiver, "create", "2015-08-12-20:00-foo", backup_files) + cmd(archiver, "create", "2015-08-12-10:00-bar", backup_files) + cmd(archiver, "create", "2015-08-12-20:00-bar", backup_files) output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:2015-*-foo") assert re.search(r"Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo", output) assert re.search(r"Would prune:\s+2015-08-12-10:00-foo", output) @@ -402,15 +402,15 @@ def test_prune_split_no_archives(): assert kept_because == {} -def test_prune_list_with_metadata_format(archivers, request): +def test_prune_list_with_metadata_format(archivers, request, backup_files): # Regression test for: prune --list with a format string that requires loading # archive metadata (e.g. {hostname}) must not fail when archives are deleted. # The bug was that format_item() was called after archive.delete(), causing # Archive.DoesNotExist when the formatter tried to lazy-load the archive. archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "test1", src_dir) - cmd(archiver, "create", "test2", src_dir) + cmd(archiver, "create", "test1", backup_files) + cmd(archiver, "create", "test2", backup_files) # {hostname} is a "call key" that triggers lazy loading of the archive from the repo. # With the buggy code this would raise Archive.DoesNotExist for the pruned archive. output = cmd(archiver, "prune", "--list", "--keep-daily=1", "--format={name} {hostname}{NL}") diff --git a/src/borg/testsuite/archiver/repo_list_cmd_test.py b/src/borg/testsuite/archiver/repo_list_cmd_test.py index 04071abff0..002225d8fd 100644 --- a/src/borg/testsuite/archiver/repo_list_cmd_test.py +++ b/src/borg/testsuite/archiver/repo_list_cmd_test.py @@ -2,28 +2,29 @@ import os from ...constants import * # NOQA -from . import cmd, checkts, create_src_archive, create_regular_file, src_dir, generate_archiver_tests, RK_ENCRYPTION +from . import cmd, checkts, create_regular_file, generate_archiver_tests, RK_ENCRYPTION +from .prune_cmd_test import _create_archive_ts pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA -def test_repo_list_glob(archivers, request): +def test_repo_list_glob(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "test-1", src_dir) - cmd(archiver, "create", "something-else-than-test-1", src_dir) - cmd(archiver, "create", "test-2", src_dir) + cmd(archiver, "create", "test-1", backup_files) + cmd(archiver, "create", "something-else-than-test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) output = cmd(archiver, "repo-list", "--match-archives=sh:test-*") assert "test-1" in output assert "test-2" in output assert "something-else" not in output -def test_archives_format(archivers, request): +def test_archives_format(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "--comment", "comment 1", "test-1", src_dir) - cmd(archiver, "create", "--comment", "comment 2", "test-2", src_dir) + cmd(archiver, "create", "--comment", "comment 1", "test-1", backup_files) + cmd(archiver, "create", "--comment", "comment 2", "test-2", backup_files) output_1 = cmd(archiver, "repo-list") output_2 = cmd( archiver, @@ -54,13 +55,13 @@ def test_size_nfiles(archivers, request): assert 123456 <= int(o_t[2]) < 123999 # There is some metadata overhead -def test_date_matching(archivers, request): +def test_date_matching(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - create_src_archive(archiver, "archive-2022-11-20", ts="2022-11-20T23:59:59") - create_src_archive(archiver, "archive-2022-12-18", ts="2022-12-18T23:59:59") - create_src_archive(archiver, "archive-now") + _create_archive_ts(archiver, backup_files, "archive-2022-11-20", 2022, 11, 20, 23, 59, 59) + _create_archive_ts(archiver, backup_files, "archive-2022-12-18", 2022, 12, 18, 23, 59, 59) + cmd(archiver, "create", "archive-now", backup_files) cmd(archiver, "check", "-v", "--oldest=23e", exit_code=2) @@ -150,13 +151,13 @@ def test_repo_list_json(archivers, request): checkts(archive0["time"]) -def test_repo_list_deleted(archivers, request): +def test_repo_list_deleted(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "normal1", src_dir) - cmd(archiver, "create", "deleted1", src_dir) - cmd(archiver, "create", "normal2", src_dir) - cmd(archiver, "create", "deleted2", src_dir) + cmd(archiver, "create", "normal1", backup_files) + cmd(archiver, "create", "deleted1", backup_files) + cmd(archiver, "create", "normal2", backup_files) + cmd(archiver, "create", "deleted2", backup_files) cmd(archiver, "delete", "-a", "sh:deleted*") output = cmd(archiver, "repo-list") assert "normal1" in output