Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 277 lines (219 sloc) 9.24 kb
902db700 »
2011-03-01 Initial commit
1 #!/usr/bin/env python
2
0642421d »
2011-03-04 Use seconds internally instead of days
3 import re
902db700 »
2011-03-01 Initial commit
4 import sys
5 import time
6 import random
7 import logging
8 import itertools
9 from optparse import OptionParser
10 from datetime import datetime, timedelta
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
11
12 import boto
13 import pytz
14 import dateutil.parser
902db700 »
2011-03-01 Initial commit
15 from boto.ec2.connection import EC2Connection
16
17 logging.basicConfig(level=logging.INFO)
18
0642421d »
2011-03-04 Use seconds internally instead of days
19 def _tdseconds(delta):
20 return delta.days * 86400 + delta.seconds
21
902db700 »
2011-03-01 Initial commit
22 def _getnow():
23 return datetime.now(tz=pytz.UTC)
24
0642421d »
2011-03-04 Use seconds internally instead of days
25 days_parse_re = re.compile('^([0-9.]+)([sMhdwmy]?)$')
448a246f »
2011-03-04 Make `--simulate` do some randomisation
26 def parse_days(days_str, single=False):
0642421d »
2011-03-04 Use seconds internally instead of days
27 specs = map(str.strip, days_str.split(','))
28 spans = []
29
30 for spec in specs:
31 spec_match = days_parse_re.match(spec)
32 if not spec_match:
33 raise Exception("Couldn't parse \"%s\"" % (spec,))
34
35 num_units = float(spec_match.group(1))
36 unit_type = spec_match.group(2)
37
38 if unit_type == 's':
39 spans.append(num_units)
40 elif unit_type == 'M':
41 spans.append(num_units*60)
42 elif unit_type == 'h':
43 spans.append(num_units*60*60)
44 elif unit_type == 'd' or unit_type == '':
45 spans.append(num_units*60*60*24)
46 elif unit_type == 'w':
47 spans.append(num_units*60*60*24*7)
48 elif unit_type == 'm':
49 spans.append(num_units*60*60*24*7*4) # n.b. a 'month' is 4 weeks
50 elif unit_type == 'y':
51 spans.append(num_units*60*60*24*7*4*12) # and a year is 12 of our 'months'
52
53 if not spans:
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
54 raise ValueError("no days specified")
0642421d »
2011-03-04 Use seconds internally instead of days
55
56 if sorted(spans) != spans:
902db700 »
2011-03-01 Initial commit
57 raise ValueError("days must be in ascending order")
0642421d »
2011-03-04 Use seconds internally instead of days
58
59 diffs = [ (spans[i] - spans[i-1]) if i != 0 else 0
60 for i in range(len(spans)) ]
61
902db700 »
2011-03-01 Initial commit
62 if sorted(diffs) != diffs:
0642421d »
2011-03-04 Use seconds internally instead of days
63 raise ValueError("diffs must be in ascending order")
64
448a246f »
2011-03-04 Make `--simulate` do some randomisation
65 if single:
66 if len(spans) != 1:
67 raise ValueError('single expected just one?')
68 return int(spans[0])
69
0642421d »
2011-03-04 Use seconds internally instead of days
70 return map(int, spans)
902db700 »
2011-03-01 Initial commit
71
72 def expire_days(days, found, key=lambda x: x):
73 # 'key' must return the number of days old that an item is
74
75 # build the list of buckets
76 buckets = []
77 for i, d in enumerate(days):
78 if i == 0:
79 start, end = 0, d
80 else:
81 start, end = buckets[-1][0][1], d
82 buckets.append(((start, end), []))
83
84 to_delete = []
85
86 # place each item in a bucket
87 for backup in found:
88 k = key(backup)
89 if k >= days[-1]:
90 # this item is older than the maximum age and should be deleted
91 to_delete.append(backup)
92 else:
93 for (start, end), backups in buckets:
94 if start <= k < end:
95 backups.append(backup)
96 break
97 else:
dd747a74 »
2011-03-08 Turn a time-drift error into a warning
98 logging.info("Ignoring backup from the future %r->%r" % (backup,k))
902db700 »
2011-03-01 Initial commit
99
100 newdays = []
101
102 for i in range(len(buckets)):
103 bucket, backups = buckets[i-len(buckets)] # in reverse order
104 if len(backups) == 1:
105 newdays.append(list(backups)[0])
106 elif len(backups) > 1:
107 backups = sorted(backups, reverse=True, key=key)
108 newdays.append(backups[0])
109 to_delete.extend(backups[1:])
110 else:
111 #logging.warning("No backups for period %r" % (bucket,))
112 pass
113
114 return newdays, to_delete
115
116 class FakeBackup(object):
117 def __init__(self, birthday):
118 self.birthday = birthday
119
120 def __repr__(self):
0642421d »
2011-03-04 Use seconds internally instead of days
121 return '<%s(%s)>' % (self.__class__.__name__,
122 self.birthday)
902db700 »
2011-03-01 Initial commit
123
448a246f »
2011-03-04 Make `--simulate` do some randomisation
124 def __eq__(self, other):
125 return self.birthday == other.birthday
0642421d »
2011-03-04 Use seconds internally instead of days
126
448a246f »
2011-03-04 Make `--simulate` do some randomisation
127 def simulate(days, ticksdiff):
128 start = now = _getnow()
902db700 »
2011-03-01 Initial commit
129
130 backups = [] # [FakeBackup(start+timedelta(days=-x)) for x in xrange(1, 100)]
131
132 print 'Starting with', days, backups
133
0642421d »
2011-03-04 Use seconds internally instead of days
134 try:
135 ticks = 0
136 while True:
448a246f »
2011-03-04 Make `--simulate` do some randomisation
137 # Amazon's snapshots take a varying amount of time so we
138 # simulate that too
139 now = start + timedelta(seconds=ticksdiff*ticks+random.randint(0, 5*60))
0642421d »
2011-03-04 Use seconds internally instead of days
140 ticks += 1
902db700 »
2011-03-01 Initial commit
141
0642421d »
2011-03-04 Use seconds internally instead of days
142 def _key(o):
143 return _tdseconds(now - o.birthday)
902db700 »
2011-03-01 Initial commit
144
0642421d »
2011-03-04 Use seconds internally instead of days
145 created = FakeBackup(now)
902db700 »
2011-03-01 Initial commit
146
0642421d »
2011-03-04 Use seconds internally instead of days
147 backups.append(created)
148 backups, deleted = expire_days(days, backups, key=_key)
448a246f »
2011-03-04 Make `--simulate` do some randomisation
149 if created in deleted:
150 # refuse to delete the one we just created like
151 # manage_snapshots does
152 logging.debug('Would have deleted the just-created %r' % (created,))
153 backups += [created]
0642421d »
2011-03-04 Use seconds internally instead of days
154 print "It's %s (%s in). %d deleted %r" % (now, now-start, len(deleted), deleted)
155 for x in sorted(backups, key=_key):
156 print "\tWe have %s: (%s old)" % (str(x), now-x.birthday)
902db700 »
2011-03-01 Initial commit
157
0642421d »
2011-03-04 Use seconds internally instead of days
158 time.sleep(0.5)
159 print '-' * 20
160 except KeyboardInterrupt:
161 return
902db700 »
2011-03-01 Initial commit
162
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
163 def manage_snapshots(days, ec2connection, vol_id, timeout=timedelta(minutes=15),
164 description='snapman'):
902db700 »
2011-03-01 Initial commit
165 volumes = ec2connection.get_all_volumes([vol_id])
166 if vol_id not in [v.id for v in volumes]:
167 raise Exception("Volume ID not found")
168
169 volume = volumes.pop()
170
171 start = _getnow()
172
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
173 descr = description + " " + start.strftime('%Y-%m-%d--%H:%M')
902db700 »
2011-03-01 Initial commit
174
175 logging.info("Creating snapshot for %r: %r" % (volume, descr))
ecb221b0 »
2011-03-03 Check that our snapshot succeeded before going off and deleting anything
176 if not volume.create_snapshot(description=descr):
177 raise Exception("Failed to create snapshot?")
902db700 »
2011-03-01 Initial commit
178
179 snapshots = volume.snapshots()
180
181 new = [ sn for sn in snapshots if sn.description == descr ]
182 if len(new) != 1:
183 raise Exception("Snapshot %r not found in %r/%r" % (descr, snapshots, new))
184 new = new.pop()
185
186 while True:
0642421d »
2011-03-04 Use seconds internally instead of days
187 # do we really need to wait for the snapshot to complete
188 # before deleting ones that it obviates? Do snapshots fail
189 # halfway through in practise?
902db700 »
2011-03-01 Initial commit
190 new.update()
191 if new.status == 'completed':
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
192 logging.info("%r completed in %s" % (new, _getnow() - start))
902db700 »
2011-03-01 Initial commit
193 break
194 elif timeout is not None and _getnow() > start + timeout:
195 raise Exception("Timed out creating %r" % (new,))
196 else:
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
197 logging.debug("Waiting for snapshot %r: %r" % (new, new.progress))
198 time.sleep(5)
902db700 »
2011-03-01 Initial commit
199
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
200 # get the new list of snapshots now that the new one is completed
902db700 »
2011-03-01 Initial commit
201 snapshots = volume.snapshots()
202
203 def _key(sn):
0642421d »
2011-03-04 Use seconds internally instead of days
204 return _tdseconds(start - dateutil.parser.parse(sn.start_time))
902db700 »
2011-03-01 Initial commit
205
206 keep, delete = expire_days(days, snapshots, key=_key)
207
208 for sn in delete:
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
209 if sn.id == new.id:
210 logging.warning("I will never delete the snapshot that I just created")
211 else:
212 logging.info("Deleting snapshot %r" % (sn,))
213 sn.delete()
902db700 »
2011-03-01 Initial commit
214
ecb221b0 »
2011-03-03 Check that our snapshot succeeded before going off and deleting anything
215 report = '\n'.join(('\t%r (%s)' % (sn, dateutil.parser.parse(sn.start_time)))
216 for sn in keep)
217 logging.info("Remaining snapshots:\n%s" % (report,))
902db700 »
2011-03-01 Initial commit
218
219 return keep, delete
220
221 def main():
0642421d »
2011-03-04 Use seconds internally instead of days
222 default_days = '1d,2d,3d,4d,5d,6d,1w,2w,3w,4w,6w,8w,12w,16w,22w'
902db700 »
2011-03-01 Initial commit
223 parser = OptionParser(usage="usage: %prog [options] vol_id")
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
224 parser.add_option('--description', default='snapman', dest='description',
225 help="prefix for snapshot description")
226 parser.add_option('--timeout', type='int', default=0, dest='timeout',
448a246f »
2011-03-04 Make `--simulate` do some randomisation
227 help="timeout for creating snapshots (see --days for units)")
902db700 »
2011-03-01 Initial commit
228 parser.add_option('--days', '-d',
229 default=default_days,
0642421d »
2011-03-04 Use seconds internally instead of days
230 help="Time spans to keep [default %default]. Units h=hours, d=days (default), w=weeks, m=months, y=years. n.b. use --simulate to make sure that your setting behaves as you think it will")
902db700 »
2011-03-01 Initial commit
231 parser.add_option("-v", "--verbose",
232 action="store_true", dest="verbose", default=False)
233 parser.add_option('--simulate', dest='simulate',
0642421d »
2011-03-04 Use seconds internally instead of days
234 help="Simulate and print the progression of backups using the given --days setting [example: --simulate=1d]")
d46222b1 »
2011-03-08 Work around a boto configuration bug with regions
235 parser.add_option('--region', dest='region', default=None,
236 help="Connect to the given EC2 region")
902db700 »
2011-03-01 Initial commit
237
238 (options, args) = parser.parse_args()
239
240 logging.basicConfig(level=logging.INFO if options.verbose else logging.WARNING)
241
242 try:
0642421d »
2011-03-04 Use seconds internally instead of days
243 days = parse_days(options.days)
244 except ValueError as e:
245 print e
902db700 »
2011-03-01 Initial commit
246 parser.print_help()
247 sys.exit(1)
248
249 if options.simulate:
448a246f »
2011-03-04 Make `--simulate` do some randomisation
250 tickspan = parse_days(options.simulate, single=True)
251 simulate(days, tickspan)
0642421d »
2011-03-04 Use seconds internally instead of days
252 sys.exit(0)
902db700 »
2011-03-01 Initial commit
253
254 if len(args) != 1:
255 parser.print_help()
256 sys.exit(1)
257 vol_id = args[0]
258
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
259 timeout=None
260 if options.timeout:
448a246f »
2011-03-04 Make `--simulate` do some randomisation
261 timeout = timedelta(seconds=parse_days(options.timeout, single=True))
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
262
902db700 »
2011-03-01 Initial commit
263 conn = EC2Connection()
d46222b1 »
2011-03-08 Work around a boto configuration bug with regions
264 if options.region is not None:
265 # this is a bit silly but we're working around a bug in boto
266 # where it half-ignores the region set in its own boto.cfg
267 # file
268 regions = dict((x.name, x)
269 for x in conn.get_all_regions())
270 region = regions[options.region]
271 conn = EC2Connection(region=region)
902db700 »
2011-03-01 Initial commit
272
9364724d »
2011-03-01 Tidy up the documentation and make it a bit more configurable
273 return manage_snapshots(days, conn, vol_id, timeout=timeout, description=options.description)
902db700 »
2011-03-01 Initial commit
274
275 if __name__ == '__main__':
276 main()
Something went wrong with that request. Please try again.