Skip to content

Commit

Permalink
Merge pull request #1: Add include/exclude filters
Browse files Browse the repository at this point in the history 
I made significant changes while merging this (e.g. the short option for
the include list and the use of shell patterns using the fnmatch module)
and I added tests to verify the behavior of the include/exclude logic.
  • Loading branch information
@xolox
xolox committed Jul 19, 2015
2 parents df3c743 + a1ceb67 commit 7771799
Show file tree
Hide file tree
Showing 4 changed files with 163 additions and 17 deletions.
14 changes: 12 additions & 2 deletions README.rst
View file
Expand Up @@ -100,6 +100,10 @@ Command line
**Usage:** `rotate-backups [OPTIONS] DIRECTORY..`

Easy rotation of backups based on the Python package by the same name. To use this program you specify a rotation scheme via (a combination of) the ``--hourly``, ``--daily``, ``--weekly``, ``--monthly`` and/or ``--yearly`` options and specify the directory (or multiple directories) containing backups to rotate as one or more positional arguments.

Please use the ``--dry-run`` option to test the effect of the specified rotation scheme before letting this program loose on your precious backups! If you don't test the results using the dry run mode and this program eats more backups than intended you have no right to complain ;-).

**Supported options:**

.. csv-table::
Expand Down Expand Up @@ -127,6 +131,14 @@ Command line
"``-y``, ``--yearly=COUNT``","Set the number of yearly backups to preserve during rotation. Refer to the
usage of the ``-H``, ``--hourly`` option for details.
"
"``-I``, ``--include=PATTERN``","Only process backups that match the shell pattern given by ``PATTERN``. This
argument can be repeated. Make sure to quote ``PATTERN`` so the shell doesn't
expand the pattern before it's received by rotate-backups.
"
"``-x``, ``--exclude=PATTERN``","Don't process backups that match the shell pattern given by ``PATTERN``. This
argument can be repeated. Make sure to quote ``PATTERN`` so the shell doesn't
expand the pattern before it's received by rotate-backups.
"
"``-i``, ``--ionice=CLASS``","Use the ""ionice"" program to set the I/O scheduling class and priority of
the ""rm"" invocations used to remove backups. ``CLASS`` is expected to be one of
the values ""idle"", ""best-effort"" or ""realtime"". Refer to the man page of
Expand All @@ -148,8 +160,6 @@ frequencies can be combined.
To-do list
----------

- Merge `pull request #1 <https://github.com/xolox/python-rotate-backups/pull/1>`_.

- Improve the Python code to make it easier to integrate into other projects as
a Python API.

Expand Down
27 changes: 23 additions & 4 deletions rotate_backups/__init__.py
View file
Expand Up @@ -5,11 +5,12 @@
# URL: https://github.com/xolox/python-rotate-backups

# Semi-standard module versioning.
__version__ = '1.0'
__version__ = '1.1'

# Standard library modules.
import collections
import datetime
import fnmatch
import functools
import logging
import os
Expand Down Expand Up @@ -47,7 +48,8 @@
''', re.VERBOSE)


def rotate_backups(directory, rotation_scheme, dry_run=False, io_scheduling_class=None):
def rotate_backups(directory, rotation_scheme, include_list=[], exclude_list=[],
dry_run=False, io_scheduling_class=None):
"""
Rotate the backups in a directory according to a flexible rotation scheme.
Expand All @@ -66,6 +68,14 @@ def rotate_backups(directory, rotation_scheme, dry_run=False, io_scheduling_clas
By default no backups are preserved for categories
(keys) not present in the dictionary.
:param include_list: A list of strings with :mod:`fnmatch` patterns. If a
nonempty include list is specified each backup must
match a pattern in the include list, otherwise it
will be ignored.
:param exclude_list: A list of strings with :mod:`fnmatch` patterns. If a
backup matches the exclude list it will be ignored,
*even if it also matched the include list* (it's the
only logical way to combine both lists).
:param dry_run: If this is ``True`` then no changes will be made, which
provides a 'preview' of the effect of the rotation scheme
(the default is ``False``). Right now this is only useful
Expand All @@ -80,10 +90,19 @@ def rotate_backups(directory, rotation_scheme, dry_run=False, io_scheduling_clas
directory = os.path.abspath(directory)
logger.info("Scanning directory for timestamped backups: %s", directory)
for entry in natsort(os.listdir(directory)):
# Check for a time stamp in the directory entry's name.
match = timestamp_pattern.search(entry)
if match:
backups.add(Backup(pathname=os.path.join(directory, entry),
datetime=datetime.datetime(*(int(group, 10) for group in match.groups('0')))))
# Make sure the entry matches the given include/exclude patterns.
if exclude_list and any(fnmatch.fnmatch(entry, p) for p in exclude_list):
logger.debug("Excluded %r (it matched the exclude list).", entry)
elif include_list and not any(fnmatch.fnmatch(entry, p) for p in include_list):
logger.debug("Excluded %r (it didn't match the include list).", entry)
else:
backups.add(Backup(
pathname=os.path.join(directory, entry),
datetime=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),
))
else:
logger.debug("Failed to match time stamp in filename: %s", entry)
if not backups:
Expand Down
55 changes: 44 additions & 11 deletions rotate_backups/cli.py
View file
Expand Up @@ -7,6 +7,17 @@
"""
Usage: rotate-backups [OPTIONS] DIRECTORY..
Easy rotation of backups based on the Python package by the same name. To use
this program you specify a rotation scheme via (a combination of) the --hourly,
--daily, --weekly, --monthly and/or --yearly options and specify the directory
(or multiple directories) containing backups to rotate as one or more
positional arguments.
Please use the --dry-run option to test the effect of the specified rotation
scheme before letting this program loose on your precious backups! If you don't
test the results using the dry run mode and this program eats more backups than
intended you have no right to complain ;-).
Supported options:
-H, --hourly=COUNT
Expand Down Expand Up @@ -39,6 +50,18 @@
Set the number of yearly backups to preserve during rotation. Refer to the
usage of the -H, --hourly option for details.
-I, --include=PATTERN
Only process backups that match the shell pattern given by PATTERN. This
argument can be repeated. Make sure to quote PATTERN so the shell doesn't
expand the pattern before it's received by rotate-backups.
-x, --exclude=PATTERN
Don't process backups that match the shell pattern given by PATTERN. This
argument can be repeated. Make sure to quote PATTERN so the shell doesn't
expand the pattern before it's received by rotate-backups.
-i, --ionice=CLASS
Use the `ionice' program to set the I/O scheduling class and priority of
Expand All @@ -60,9 +83,6 @@
Show this message and exit.
"""

# Semi-standard module versioning.
__version__ = '0.1.2'

# Standard library modules.
import getopt
import logging
Expand All @@ -84,19 +104,22 @@

def main():
"""Command line interface for the ``rotate-backups`` program."""
dry_run = False
io_scheduling_class = None
rotation_scheme = {}
coloredlogs.install()
if os.path.exists('/dev/log'):
handler = logging.handlers.SysLogHandler(address='/dev/log')
handler.setFormatter(logging.Formatter('%(module)s[%(process)d] %(levelname)s %(message)s'))
logger.addHandler(handler)
# Command line option defaults.
dry_run = False
exclude_list = []
include_list = []
io_scheduling_class = None
rotation_scheme = {}
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:i:nvh', [
'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'ionice=',
'dry-run', 'verbose', 'help'
options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:I:x:i:nvh', [
'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=',
'exclude=', 'ionice=', 'dry-run', 'verbose', 'help',
])
for option, value in options:
if option in ('-H', '--hourly'):
Expand All @@ -109,6 +132,10 @@ def main():
rotation_scheme['monthly'] = cast_to_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = cast_to_retention_period(value)
elif option in ('-I', '--include'):
include_list.append(value)
elif option in ('-x', '--exclude'):
exclude_list.append(value)
elif option in ('-i', '--ionice'):
value = value.lower().strip()
expected = ('idle', 'best-effort', 'realtime')
Expand Down Expand Up @@ -139,8 +166,14 @@ def main():
sys.exit(1)
# Rotate the backups in the given directories.
for pathname in arguments:
rotate_backups(pathname, rotation_scheme, dry_run=dry_run,
io_scheduling_class=io_scheduling_class)
rotate_backups(
directory=pathname,
rotation_scheme=rotation_scheme,
include_list=include_list,
exclude_list=exclude_list,
io_scheduling_class=io_scheduling_class,
dry_run=dry_run,
)


def cast_to_retention_period(value):
Expand Down
84 changes: 84 additions & 0 deletions rotate_backups/tests.py
View file
Expand Up @@ -146,6 +146,90 @@ def test_rotate_backups(self):
backups_that_were_preserved = set(os.listdir(root))
assert backups_that_were_preserved == expected_to_be_preserved

def test_include_list(self):
"""Test include list logic."""
# These are the backups expected to be preserved within the year 2014
# (other years are excluded and so should all be preserved, see below).
# After each backup I've noted which rotation scheme it falls in.
expected_to_be_preserved = set([
'2014-01-01@20:07', # monthly, yearly
'2014-02-01@20:05', # monthly
'2014-03-01@20:04', # monthly
'2014-04-01@20:03', # monthly
'2014-05-01@20:06', # monthly
'2014-05-19@20:02', # weekly
'2014-05-26@20:05', # weekly
'2014-06-01@20:01', # monthly
'2014-06-02@20:05', # weekly
'2014-06-09@20:01', # weekly
'2014-06-16@20:02', # weekly
'2014-06-23@20:04', # weekly
'2014-06-26@20:04', # daily
'2014-06-27@20:02', # daily
'2014-06-28@20:02', # daily
'2014-06-29@20:01', # daily
'2014-06-30@20:03', # daily, weekly
'2014-07-01@20:02', # daily, monthly
'2014-07-02@20:03', # hourly, daily
'some-random-directory', # no recognizable time stamp, should definitely be preserved
])
for name in SAMPLE_BACKUP_SET:
if not name.startswith('2014-'):
expected_to_be_preserved.add(name)
with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root:
self.create_sample_backup_set(root)
run_cli(
'--verbose', '--ionice=idle', '--hourly=24', '--daily=7',
'--weekly=7', '--monthly=12', '--yearly=always',
'--include=2014-*', root,
)
backups_that_were_preserved = set(os.listdir(root))
assert backups_that_were_preserved == expected_to_be_preserved

def test_exclude_list(self):
"""Test exclude list logic."""
# These are the backups expected to be preserved. After each backup
# I've noted which rotation scheme it falls in and the number of
# preserved backups within that rotation scheme (counting up as we
# progress through the backups sorted by date).
expected_to_be_preserved = set([
'2013-10-10@20:07', # monthly (1), yearly (1)
'2013-11-01@20:06', # monthly (2)
'2013-12-01@20:07', # monthly (3)
'2014-01-01@20:07', # monthly (4), yearly (2)
'2014-02-01@20:05', # monthly (5)
'2014-03-01@20:04', # monthly (6)
'2014-04-01@20:03', # monthly (7)
'2014-05-01@20:06', # monthly (8)
'2014-05-19@20:02', # weekly (1)
'2014-05-26@20:05', # weekly (2)
'2014-06-01@20:01', # monthly (9)
'2014-06-02@20:05', # weekly (3)
'2014-06-09@20:01', # weekly (4)
'2014-06-16@20:02', # weekly (5)
'2014-06-23@20:04', # weekly (6)
'2014-06-26@20:04', # daily (1)
'2014-06-27@20:02', # daily (2)
'2014-06-28@20:02', # daily (3)
'2014-06-29@20:01', # daily (4)
'2014-06-30@20:03', # daily (5), weekly (7)
'2014-07-01@20:02', # daily (6), monthly (10)
'2014-07-02@20:03', # hourly (1), daily (7)
'some-random-directory', # no recognizable time stamp, should definitely be preserved
])
for name in SAMPLE_BACKUP_SET:
if name.startswith('2014-05-'):
expected_to_be_preserved.add(name)
with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root:
self.create_sample_backup_set(root)
run_cli(
'--verbose', '--ionice=idle', '--hourly=24', '--daily=7',
'--weekly=7', '--monthly=12', '--yearly=always',
'--exclude=2014-05-*', root,
)
backups_that_were_preserved = set(os.listdir(root))
assert backups_that_were_preserved == expected_to_be_preserved

def create_sample_backup_set(self, root):
"""Create a sample backup set to be rotated."""
for name in SAMPLE_BACKUP_SET:
Expand Down

0 comments on commit 7771799

Please sign in to comment.