diff --git a/README.rst b/README.rst index 8db27d7..167c711 100644 --- a/README.rst +++ b/README.rst @@ -100,6 +100,10 @@ Command line **Usage:** `rotate-backups [OPTIONS] DIRECTORY..` +Easy rotation of backups based on the Python package by the same name. To use this program you specify a rotation scheme via (a combination of) the ``--hourly``, ``--daily``, ``--weekly``, ``--monthly`` and/or ``--yearly`` options and specify the directory (or multiple directories) containing backups to rotate as one or more positional arguments. + +Please use the ``--dry-run`` option to test the effect of the specified rotation scheme before letting this program loose on your precious backups! If you don't test the results using the dry run mode and this program eats more backups than intended you have no right to complain ;-). + **Supported options:** .. csv-table:: @@ -127,6 +131,14 @@ Command line "``-y``, ``--yearly=COUNT``","Set the number of yearly backups to preserve during rotation. Refer to the usage of the ``-H``, ``--hourly`` option for details. " + "``-I``, ``--include=PATTERN``","Only process backups that match the shell pattern given by ``PATTERN``. This + argument can be repeated. Make sure to quote ``PATTERN`` so the shell doesn't + expand the pattern before it's received by rotate-backups. + " + "``-x``, ``--exclude=PATTERN``","Don't process backups that match the shell pattern given by ``PATTERN``. This + argument can be repeated. Make sure to quote ``PATTERN`` so the shell doesn't + expand the pattern before it's received by rotate-backups. + " "``-i``, ``--ionice=CLASS``","Use the ""ionice"" program to set the I/O scheduling class and priority of the ""rm"" invocations used to remove backups. ``CLASS`` is expected to be one of the values ""idle"", ""best-effort"" or ""realtime"". Refer to the man page of @@ -148,8 +160,6 @@ frequencies can be combined. To-do list ---------- -- Merge `pull request #1 `_. - - Improve the Python code to make it easier to integrate into other projects as a Python API. diff --git a/rotate_backups/__init__.py b/rotate_backups/__init__.py index 08409c8..33ec3bc 100644 --- a/rotate_backups/__init__.py +++ b/rotate_backups/__init__.py @@ -5,11 +5,12 @@ # URL: https://github.com/xolox/python-rotate-backups # Semi-standard module versioning. -__version__ = '1.0' +__version__ = '1.1' # Standard library modules. import collections import datetime +import fnmatch import functools import logging import os @@ -47,7 +48,8 @@ ''', re.VERBOSE) -def rotate_backups(directory, rotation_scheme, dry_run=False, io_scheduling_class=None): +def rotate_backups(directory, rotation_scheme, include_list=[], exclude_list=[], + dry_run=False, io_scheduling_class=None): """ Rotate the backups in a directory according to a flexible rotation scheme. @@ -66,6 +68,14 @@ def rotate_backups(directory, rotation_scheme, dry_run=False, io_scheduling_clas By default no backups are preserved for categories (keys) not present in the dictionary. + :param include_list: A list of strings with :mod:`fnmatch` patterns. If a + nonempty include list is specified each backup must + match a pattern in the include list, otherwise it + will be ignored. + :param exclude_list: A list of strings with :mod:`fnmatch` patterns. If a + backup matches the exclude list it will be ignored, + *even if it also matched the include list* (it's the + only logical way to combine both lists). :param dry_run: If this is ``True`` then no changes will be made, which provides a 'preview' of the effect of the rotation scheme (the default is ``False``). Right now this is only useful @@ -80,10 +90,19 @@ def rotate_backups(directory, rotation_scheme, dry_run=False, io_scheduling_clas directory = os.path.abspath(directory) logger.info("Scanning directory for timestamped backups: %s", directory) for entry in natsort(os.listdir(directory)): + # Check for a time stamp in the directory entry's name. match = timestamp_pattern.search(entry) if match: - backups.add(Backup(pathname=os.path.join(directory, entry), - datetime=datetime.datetime(*(int(group, 10) for group in match.groups('0'))))) + # Make sure the entry matches the given include/exclude patterns. + if exclude_list and any(fnmatch.fnmatch(entry, p) for p in exclude_list): + logger.debug("Excluded %r (it matched the exclude list).", entry) + elif include_list and not any(fnmatch.fnmatch(entry, p) for p in include_list): + logger.debug("Excluded %r (it didn't match the include list).", entry) + else: + backups.add(Backup( + pathname=os.path.join(directory, entry), + datetime=datetime.datetime(*(int(group, 10) for group in match.groups('0'))), + )) else: logger.debug("Failed to match time stamp in filename: %s", entry) if not backups: diff --git a/rotate_backups/cli.py b/rotate_backups/cli.py index b1e7b94..2340df7 100644 --- a/rotate_backups/cli.py +++ b/rotate_backups/cli.py @@ -7,6 +7,17 @@ """ Usage: rotate-backups [OPTIONS] DIRECTORY.. +Easy rotation of backups based on the Python package by the same name. To use +this program you specify a rotation scheme via (a combination of) the --hourly, +--daily, --weekly, --monthly and/or --yearly options and specify the directory +(or multiple directories) containing backups to rotate as one or more +positional arguments. + +Please use the --dry-run option to test the effect of the specified rotation +scheme before letting this program loose on your precious backups! If you don't +test the results using the dry run mode and this program eats more backups than +intended you have no right to complain ;-). + Supported options: -H, --hourly=COUNT @@ -39,6 +50,18 @@ Set the number of yearly backups to preserve during rotation. Refer to the usage of the -H, --hourly option for details. + -I, --include=PATTERN + + Only process backups that match the shell pattern given by PATTERN. This + argument can be repeated. Make sure to quote PATTERN so the shell doesn't + expand the pattern before it's received by rotate-backups. + + -x, --exclude=PATTERN + + Don't process backups that match the shell pattern given by PATTERN. This + argument can be repeated. Make sure to quote PATTERN so the shell doesn't + expand the pattern before it's received by rotate-backups. + -i, --ionice=CLASS Use the `ionice' program to set the I/O scheduling class and priority of @@ -60,9 +83,6 @@ Show this message and exit. """ -# Semi-standard module versioning. -__version__ = '0.1.2' - # Standard library modules. import getopt import logging @@ -84,19 +104,22 @@ def main(): """Command line interface for the ``rotate-backups`` program.""" - dry_run = False - io_scheduling_class = None - rotation_scheme = {} coloredlogs.install() if os.path.exists('/dev/log'): handler = logging.handlers.SysLogHandler(address='/dev/log') handler.setFormatter(logging.Formatter('%(module)s[%(process)d] %(levelname)s %(message)s')) logger.addHandler(handler) + # Command line option defaults. + dry_run = False + exclude_list = [] + include_list = [] + io_scheduling_class = None + rotation_scheme = {} # Parse the command line arguments. try: - options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:i:nvh', [ - 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'ionice=', - 'dry-run', 'verbose', 'help' + options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:I:x:i:nvh', [ + 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=', + 'exclude=', 'ionice=', 'dry-run', 'verbose', 'help', ]) for option, value in options: if option in ('-H', '--hourly'): @@ -109,6 +132,10 @@ def main(): rotation_scheme['monthly'] = cast_to_retention_period(value) elif option in ('-y', '--yearly'): rotation_scheme['yearly'] = cast_to_retention_period(value) + elif option in ('-I', '--include'): + include_list.append(value) + elif option in ('-x', '--exclude'): + exclude_list.append(value) elif option in ('-i', '--ionice'): value = value.lower().strip() expected = ('idle', 'best-effort', 'realtime') @@ -139,8 +166,14 @@ def main(): sys.exit(1) # Rotate the backups in the given directories. for pathname in arguments: - rotate_backups(pathname, rotation_scheme, dry_run=dry_run, - io_scheduling_class=io_scheduling_class) + rotate_backups( + directory=pathname, + rotation_scheme=rotation_scheme, + include_list=include_list, + exclude_list=exclude_list, + io_scheduling_class=io_scheduling_class, + dry_run=dry_run, + ) def cast_to_retention_period(value): diff --git a/rotate_backups/tests.py b/rotate_backups/tests.py index 1fb8281..344db09 100644 --- a/rotate_backups/tests.py +++ b/rotate_backups/tests.py @@ -146,6 +146,90 @@ def test_rotate_backups(self): backups_that_were_preserved = set(os.listdir(root)) assert backups_that_were_preserved == expected_to_be_preserved + def test_include_list(self): + """Test include list logic.""" + # These are the backups expected to be preserved within the year 2014 + # (other years are excluded and so should all be preserved, see below). + # After each backup I've noted which rotation scheme it falls in. + expected_to_be_preserved = set([ + '2014-01-01@20:07', # monthly, yearly + '2014-02-01@20:05', # monthly + '2014-03-01@20:04', # monthly + '2014-04-01@20:03', # monthly + '2014-05-01@20:06', # monthly + '2014-05-19@20:02', # weekly + '2014-05-26@20:05', # weekly + '2014-06-01@20:01', # monthly + '2014-06-02@20:05', # weekly + '2014-06-09@20:01', # weekly + '2014-06-16@20:02', # weekly + '2014-06-23@20:04', # weekly + '2014-06-26@20:04', # daily + '2014-06-27@20:02', # daily + '2014-06-28@20:02', # daily + '2014-06-29@20:01', # daily + '2014-06-30@20:03', # daily, weekly + '2014-07-01@20:02', # daily, monthly + '2014-07-02@20:03', # hourly, daily + 'some-random-directory', # no recognizable time stamp, should definitely be preserved + ]) + for name in SAMPLE_BACKUP_SET: + if not name.startswith('2014-'): + expected_to_be_preserved.add(name) + with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: + self.create_sample_backup_set(root) + run_cli( + '--verbose', '--ionice=idle', '--hourly=24', '--daily=7', + '--weekly=7', '--monthly=12', '--yearly=always', + '--include=2014-*', root, + ) + backups_that_were_preserved = set(os.listdir(root)) + assert backups_that_were_preserved == expected_to_be_preserved + + def test_exclude_list(self): + """Test exclude list logic.""" + # These are the backups expected to be preserved. After each backup + # I've noted which rotation scheme it falls in and the number of + # preserved backups within that rotation scheme (counting up as we + # progress through the backups sorted by date). + expected_to_be_preserved = set([ + '2013-10-10@20:07', # monthly (1), yearly (1) + '2013-11-01@20:06', # monthly (2) + '2013-12-01@20:07', # monthly (3) + '2014-01-01@20:07', # monthly (4), yearly (2) + '2014-02-01@20:05', # monthly (5) + '2014-03-01@20:04', # monthly (6) + '2014-04-01@20:03', # monthly (7) + '2014-05-01@20:06', # monthly (8) + '2014-05-19@20:02', # weekly (1) + '2014-05-26@20:05', # weekly (2) + '2014-06-01@20:01', # monthly (9) + '2014-06-02@20:05', # weekly (3) + '2014-06-09@20:01', # weekly (4) + '2014-06-16@20:02', # weekly (5) + '2014-06-23@20:04', # weekly (6) + '2014-06-26@20:04', # daily (1) + '2014-06-27@20:02', # daily (2) + '2014-06-28@20:02', # daily (3) + '2014-06-29@20:01', # daily (4) + '2014-06-30@20:03', # daily (5), weekly (7) + '2014-07-01@20:02', # daily (6), monthly (10) + '2014-07-02@20:03', # hourly (1), daily (7) + 'some-random-directory', # no recognizable time stamp, should definitely be preserved + ]) + for name in SAMPLE_BACKUP_SET: + if name.startswith('2014-05-'): + expected_to_be_preserved.add(name) + with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: + self.create_sample_backup_set(root) + run_cli( + '--verbose', '--ionice=idle', '--hourly=24', '--daily=7', + '--weekly=7', '--monthly=12', '--yearly=always', + '--exclude=2014-05-*', root, + ) + backups_that_were_preserved = set(os.listdir(root)) + assert backups_that_were_preserved == expected_to_be_preserved + def create_sample_backup_set(self, root): """Create a sample backup set to be rotated.""" for name in SAMPLE_BACKUP_SET: