diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index c6d91f92b4..fd24a1d93f 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -19,7 +19,7 @@ from errno import EEXIST from gzip import GzipFile from os import mkdir from os.path import basename, dirname, exists, join as pathjoin -from sys import argv, exit +from sys import argv, exit, modules from textwrap import wrap from time import time @@ -153,9 +153,9 @@ swift-ring-builder create except OSError, err: if err.errno != EEXIST: raise - pickle.dump(builder, open(pathjoin(backup_dir, + pickle.dump(builder.to_dict(), open(pathjoin(backup_dir, '%d.' % time() + basename(argv[1])), 'wb'), protocol=2) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_CHANGED) def default(): @@ -312,7 +312,7 @@ swift-ring-builder add z-:/_ else: print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ (zone, ip, port, device_name, meta, weight, next_dev_id) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def set_weight(): @@ -345,7 +345,7 @@ swift-ring-builder set_weight builder.set_dev_weight(dev['id'], weight) print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ 'weight set to %(weight)s' % dev - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def set_info(): @@ -427,7 +427,7 @@ swift-ring-builder set_info for key, value in change: dev[key] = value print 'Device %s is now %s' % (orig_dev_string, format_device(dev)) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def remove(): @@ -463,7 +463,7 @@ swift-ring-builder remove print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s" ' \ 'marked for removal and will be removed next rebalance.' \ % dev - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def rebalance(): @@ -495,13 +495,14 @@ swift-ring-builder rebalance % builder.min_part_hours print '-' * 79 ts = time() - pickle.dump(builder.get_ring(), + pickle.dump(builder.get_ring().to_dict(), GzipFile(pathjoin(backup_dir, '%d.' % ts + basename(ring_file)), 'wb'), protocol=2) - pickle.dump(builder, open(pathjoin(backup_dir, + pickle.dump(builder.to_dict(), open(pathjoin(backup_dir, '%d.' % ts + basename(argv[1])), 'wb'), protocol=2) - pickle.dump(builder.get_ring(), GzipFile(ring_file, 'wb'), protocol=2) - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.get_ring().to_dict(), GzipFile(ring_file, 'wb'), + protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_CHANGED) def validate(): @@ -528,15 +529,15 @@ swift-ring-builder write_ring '"rebalance"?' else: print 'Warning: Writing an empty ring' - pickle.dump(ring_data, + pickle.dump(ring_data.to_dict(), GzipFile(pathjoin(backup_dir, '%d.' % time() + basename(ring_file)), 'wb'), protocol=2) - pickle.dump(ring_data, GzipFile(ring_file, 'wb'), protocol=2) + pickle.dump(ring_data.to_dict(), GzipFile(ring_file, 'wb'), protocol=2) exit(EXIT_RING_CHANGED) def pretend_min_part_hours_passed(): builder.pretend_min_part_hours_passed() - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) def set_min_part_hours(): @@ -552,7 +553,7 @@ swift-ring-builder set_min_part_hours builder.change_min_part_hours(int(argv[3])) print 'The minimum number of hours before a partition can be ' \ 'reassigned is now set to %s' % argv[3] - pickle.dump(builder, open(argv[1], 'wb'), protocol=2) + pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_RING_UNCHANGED) @@ -578,7 +579,17 @@ if __name__ == '__main__': exit(EXIT_RING_UNCHANGED) if exists(argv[1]): - builder = pickle.load(open(argv[1], 'rb')) + try: + builder = pickle.load(open(argv[1], 'rb')) + if not hasattr(builder, 'devs'): + builder_dict = builder + builder = RingBuilder(1, 1, 1) + builder.copy_from(builder_dict) + except ImportError: # Happens with really old builder pickles + modules['swift.ring_builder'] = \ + modules['swift.common.ring.builder'] + builder = RingBuilder(1, 1, 1) + builder.copy_from(pickle.load(open(argv[1], 'rb'))) for dev in builder.devs: if dev and 'meta' not in dev: dev['meta'] = '' diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py index 3f728e307a..86e6cce287 100644 --- a/swift/common/ring/builder.py +++ b/swift/common/ring/builder.py @@ -69,6 +69,49 @@ def __init__(self, part_power, replicas, min_part_hours): self._remove_devs = [] self._ring = None + def copy_from(self, builder): + if hasattr(builder, 'devs'): + self.part_power = builder.part_power + self.replicas = builder.replicas + self.min_part_hours = builder.min_part_hours + self.parts = builder.parts + self.devs = builder.devs + self.devs_changed = builder.devs_changed + self.version = builder.version + self._replica2part2dev = builder._replica2part2dev + self._last_part_moves_epoch = builder._last_part_moves_epoch + self._last_part_moves = builder._last_part_moves + self._last_part_gather_start = builder._last_part_gather_start + self._remove_devs = builder._remove_devs + else: + self.part_power = builder['part_power'] + self.replicas = builder['replicas'] + self.min_part_hours = builder['min_part_hours'] + self.parts = builder['parts'] + self.devs = builder['devs'] + self.devs_changed = builder['devs_changed'] + self.version = builder['version'] + self._replica2part2dev = builder['_replica2part2dev'] + self._last_part_moves_epoch = builder['_last_part_moves_epoch'] + self._last_part_moves = builder['_last_part_moves'] + self._last_part_gather_start = builder['_last_part_gather_start'] + self._remove_devs = builder['_remove_devs'] + self._ring = None + + def to_dict(self): + return {'part_power': self.part_power, + 'replicas': self.replicas, + 'min_part_hours': self.min_part_hours, + 'parts': self.parts, + 'devs': self.devs, + 'devs_changed': self.devs_changed, + 'version': self.version, + '_replica2part2dev': self._replica2part2dev, + '_last_part_moves_epoch': self._last_part_moves_epoch, + '_last_part_moves': self._last_part_moves, + '_last_part_gather_start': self._last_part_gather_start, + '_remove_devs': self._remove_devs} + def change_min_part_hours(self, min_part_hours): """ Changes the value used to decide if a given partition can be moved diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 45ab407563..122a990fc5 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -29,6 +29,11 @@ def __init__(self, replica2part2dev_id, devs, part_shift): self._replica2part2dev_id = replica2part2dev_id self._part_shift = part_shift + def to_dict(self): + return {'devs': self.devs, + 'replica2part2dev_id': self._replica2part2dev_id, + 'part_shift': self._part_shift} + class Ring(object): """ @@ -47,6 +52,9 @@ def _reload(self, force=False): self._rtime = time() + self.reload_time if force or self.has_changed(): ring_data = pickle.load(GzipFile(self.pickle_gz_path, 'rb')) + if not hasattr(ring_data, 'devs'): + ring_data = RingData(ring_data['replica2part2dev_id'], + ring_data['devs'], ring_data['part_shift']) self._mtime = getmtime(self.pickle_gz_path) self.devs = ring_data.devs self.zone2devs = {}