Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Use seconds internally instead of days

  • Loading branch information...
commit 0642421d8aa78d323c47e017ac55728e48685e80 1 parent ecb221b
@ketralnis authored
Showing with 102 additions and 46 deletions.
  1. +23 −13 README.md
  2. +79 −33 snapman.py
View
36 README.md
@@ -1,16 +1,25 @@
snapman: a simple EC2 snapshot manager
======================================
-Creates EC2 snapshots of the given EBS volume and keeps around a
-configurable set of snapshots that are at oldest X days
+Creates EC2 snapshots of a given EBS volume and keeps around a
+configurable set phased over increasing intervals
Example
--------
- snapman.py --days 1,2,3,4,5,6,7,14,21 vol-abcde
+ snapman.py --days 1,2,3,4,5,6,1w,2w,3w vol-abcde
-This takes a snapshot of vol-abcde and deletes all but one snapshot per
-day for the last week, plus one at most 2 and 3 weeks old
+This takes a snapshot of `vol-abcde` and deletes all but one snapshot
+per day for the last week, plus two more at 2 and 3 weeks old
+
+Configuration
+-------------
+
+* for EC2 credentials, use boto's configuration files or environment
+ variables <http://code.google.com/p/boto/wiki/BotoConfig>
+* if you're snapshotting a volume that's mounted, make sure to wrap
+ snapman in a script that makes the snapshot consistent (e.g.
+ `xfs_freeze` or `pg_start_backup`)
Requirements
------------
@@ -20,15 +29,16 @@ Requirements
* pytz
* python-dateutil
-Configuration
--------------
-
-use boto's configuration files or environment variables <http://code.google.com/p/boto/wiki/BotoConfig>
-
To do
-----
-* don't delete snapshots that we didn't create (perhaps by checking the description prefix)
-* mount snapshots at /${mountpoint}/.snap/${timestamp}/
-
+* don't delete snapshots that we didn't create, or at least make
+ that the default (perhaps by checking the description prefix)
+* keep snapshots mounted and accessible at
+ `/${mountpoint}/.snap/${timestamp}/` (can we do this without
+ creating volumes out of every snapshot?)
+* options to only create snapshot, only expire snapshots, and only
+ print what snapshots would be deleted
+* detect and warn that the given `--days` setting can't keep all
+ of the requested backups instead of relying on `--simulate`
View
112 snapman.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+import re
import sys
import time
import random
@@ -15,18 +16,53 @@
logging.basicConfig(level=logging.INFO)
+def _tdseconds(delta):
+ return delta.days * 86400 + delta.seconds
+
def _getnow():
return datetime.now(tz=pytz.UTC)
-def _validate_days(days):
- if not days:
+days_parse_re = re.compile('^([0-9.]+)([sMhdwmy]?)$')
+def parse_days(days_str):
+ specs = map(str.strip, days_str.split(','))
+ spans = []
+
+ for spec in specs:
+ spec_match = days_parse_re.match(spec)
+ if not spec_match:
+ raise Exception("Couldn't parse \"%s\"" % (spec,))
+
+ num_units = float(spec_match.group(1))
+ unit_type = spec_match.group(2)
+
+ if unit_type == 's':
+ spans.append(num_units)
+ elif unit_type == 'M':
+ spans.append(num_units*60)
+ elif unit_type == 'h':
+ spans.append(num_units*60*60)
+ elif unit_type == 'd' or unit_type == '':
+ spans.append(num_units*60*60*24)
+ elif unit_type == 'w':
+ spans.append(num_units*60*60*24*7)
+ elif unit_type == 'm':
+ spans.append(num_units*60*60*24*7*4) # n.b. a 'month' is 4 weeks
+ elif unit_type == 'y':
+ spans.append(num_units*60*60*24*7*4*12) # and a year is 12 of our 'months'
+
+ if not spans:
raise ValueError("no days specified")
- if sorted(days) != days:
+
+ if sorted(spans) != spans:
raise ValueError("days must be in ascending order")
- diffs = [ (days[i] - days[i-1]) if i != 0 else 0
- for i in range(len(days)) ]
+
+ diffs = [ (spans[i] - spans[i-1]) if i != 0 else 0
+ for i in range(len(spans)) ]
+
if sorted(diffs) != diffs:
- raise ValueError("diffs must be in ascending order")
+ raise ValueError("diffs must be in ascending order")
+
+ return map(int, spans)
def expire_days(days, found, key=lambda x: x):
# 'key' must return the number of days old that an item is
@@ -77,34 +113,41 @@ def __init__(self, birthday):
self.birthday = birthday
def __repr__(self):
- return '<%s(%s)>' % (self.__class__.__name__, self.birthday.date())
+ return '<%s(%s)>' % (self.__class__.__name__,
+ self.birthday)
-def simulate(days):
- _validate_days(days)
-
- start = _getnow()
+def simulate(days, tickspan):
+ start = now = _getnow()
+
+ ticks = parse_days(tickspan)
+ assert len(ticks) == 1
+ ticksdiff = timedelta(seconds=ticks[0])
backups = [] # [FakeBackup(start+timedelta(days=-x)) for x in xrange(1, 100)]
print 'Starting with', days, backups
- ticks = 0
- while True:
- now = start + timedelta(days=ticks)
- ticks += 1
+ try:
+ ticks = 0
+ while True:
+ now += ticksdiff
+ ticks += 1
- def _key(o):
- return (now - o.birthday).days
+ def _key(o):
+ return _tdseconds(now - o.birthday)
- created = FakeBackup(now)
+ created = FakeBackup(now)
- backups.append(created)
- backups, deleted = expire_days(days, backups, key=_key)
- print "It's %s (%d days in). %d deleted %r" % (now.date(), ticks, len(deleted), deleted)
- for x in sorted(backups, key=_key):
- print "\tWe have %s: (%d days old)" % (str(x), (now-x.birthday).days)
+ backups.append(created)
+ backups, deleted = expire_days(days, backups, key=_key)
+ print "It's %s (%s in). %d deleted %r" % (now, now-start, len(deleted), deleted)
+ for x in sorted(backups, key=_key):
+ print "\tWe have %s: (%s old)" % (str(x), now-x.birthday)
- time.sleep(0.5)
+ time.sleep(0.5)
+ print '-' * 20
+ except KeyboardInterrupt:
+ return
def manage_snapshots(days, ec2connection, vol_id, timeout=timedelta(minutes=15),
description='snapman'):
@@ -130,6 +173,9 @@ def manage_snapshots(days, ec2connection, vol_id, timeout=timedelta(minutes=15),
new = new.pop()
while True:
+ # do we really need to wait for the snapshot to complete
+ # before deleting ones that it obviates? Do snapshots fail
+ # halfway through in practise?
new.update()
if new.status == 'completed':
logging.info("%r completed in %s" % (new, _getnow() - start))
@@ -144,7 +190,7 @@ def manage_snapshots(days, ec2connection, vol_id, timeout=timedelta(minutes=15),
snapshots = volume.snapshots()
def _key(sn):
- return (start - dateutil.parser.parse(sn.start_time)).days
+ return _tdseconds(start - dateutil.parser.parse(sn.start_time))
keep, delete = expire_days(days, snapshots, key=_key)
@@ -162,7 +208,7 @@ def _key(sn):
return keep, delete
def main():
- default_days = '1,2,3,4,5,6,7,14,21,28,42,56,84,112'
+ default_days = '1d,2d,3d,4d,5d,6d,1w,2w,3w,4w,6w,8w,12w,16w,22w'
parser = OptionParser(usage="usage: %prog [options] vol_id")
parser.add_option('--description', default='snapman', dest='description',
help="prefix for snapshot description")
@@ -170,26 +216,26 @@ def main():
help="timeout in minutes for creating snapshot")
parser.add_option('--days', '-d',
default=default_days,
- help="Day spans to keep [default %default]")
+ help="Time spans to keep [default %default]. Units h=hours, d=days (default), w=weeks, m=months, y=years. n.b. use --simulate to make sure that your setting behaves as you think it will")
parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False)
parser.add_option('--simulate', dest='simulate',
- default=False, action='store_true',
- help="Simulate and print the progression of backups using the given --days setting")
+ help="Simulate and print the progression of backups using the given --days setting [example: --simulate=1d]")
(options, args) = parser.parse_args()
logging.basicConfig(level=logging.INFO if options.verbose else logging.WARNING)
try:
- days = map(int, options.days.split(','))
- _validate_days(days)
- except ValueError:
+ days = parse_days(options.days)
+ except ValueError as e:
+ print e
parser.print_help()
sys.exit(1)
if options.simulate:
- return simulate(days)
+ simulate(days, options.simulate)
+ sys.exit(0)
if len(args) != 1:
parser.print_help()
Please sign in to comment.
Something went wrong with that request. Please try again.