Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

testing (fixing) persistence

  • Loading branch information...
commit 62d82fff646d6c331fb5d416721f5fe79668e915 1 parent 00e9fb4
@truthtrap truthtrap authored
View
2  administration.py
@@ -167,7 +167,7 @@ def get_expired_snapshots(key, access, cluster):
now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
select = "select * from `{0}` where itemName() > 'snap-' and itemName() != 'snapshot' and expires < '{1}'".format(cluster, now)
- snapshots = domain.select(select)
+ snapshots = domain.select(select, consistent_read=True)
return snapshots
def get_identity(key, access, cluster):
View
2  backup.py
@@ -79,7 +79,7 @@ def create_bucket(key, access, cluster):
try:
s3.create_bucket( cluster.replace('.', '-'), location=Location.EU)
except S3CreateError as e:
- print e
+ pass
def put_RDB(key, access, cluster, prefix='hourly'):
s3 = S3Connection(key, access)
View
4 client.py
@@ -72,11 +72,11 @@ def get_slave(self, node=None):
def size(self):
select = "select count(*) from `{0}` where itemName() like '%.{0}'".format(self.domain.name)
- return int(self.domain.select(select).next()['Count'])
+ return int(self.domain.select(select, consistent_read=True).next()['Count'])
def slaves(self):
select = "select itemName() from `{0}` where master is not null and master != '' and itemName() like '%.{0}'".format(self.domain.name)
- items = self.domain.select(select)
+ items = self.domain.select(select, consistent_read=True)
slaves = []
for item in items:
View
2  cluster.py
@@ -205,7 +205,7 @@ def get_slave(self, node=None):
def size(self):
select = "select count(*) from `{0}` where itemName() like '%.{0}'".format(self.domain.name)
- return int(self.domain.select(select).next()['Count'])
+ return int(self.domain.select(select, consistent_read=True).next()['Count'])
def check_integrity(self, cluster):
pass
View
29 decommission.py
@@ -41,10 +41,9 @@ def decommission(key, access, cluster, persistence="no"):
rdb = backup.put_RDB(key, access, cluster, 'monthly')
administration.set_RDB(key, access, cluster, rdb)
- # if we have a volume, make a last snapshot
- if "low" != persistence:
- snapshot = backup.make_snapshot(key, access, cluster, 'monthly')
- administration.add_snapshot(key, access, cluster, snapshot)
+ # make a last snapshot
+ snapshot = backup.make_snapshot(key, access, cluster, 'monthly')
+ administration.add_snapshot(key, access, cluster, snapshot)
# we don't have to get rid any the volume, it is deleted on termination
@@ -54,18 +53,18 @@ def decommission(key, access, cluster, persistence="no"):
os.system("/bin/echo | /usr/bin/crontab")
if __name__ == '__main__':
- import os, sys
+ import os, sys
- try:
- persistence = userdata['persistence']
- except:
- persistence = None
+ try:
+ persistence = userdata['persistence']
+ except:
+ persistence = None
- # what is the domain to work with
- name = os.environ['REDIS_NAME'].strip()
- zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
+ # what is the domain to work with
+ name = os.environ['REDIS_NAME'].strip()
+ zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
- # the name (and identity) of the cluster (the master)
- cluster = "{0}.{1}".format(name, zone)
+ # the name (and identity) of the cluster (the master)
+ cluster = "{0}.{1}".format(name, zone)
- decommission(sys.argv[1], sys.argv[2], cluster, persistence=persistence)
+ decommission(sys.argv[1], sys.argv[2], cluster, persistence=persistence)
View
15 host.py
@@ -56,10 +56,17 @@ def get_master(self):
def set_master(self, master=None):
self.master = master
- if None == master:
- self.redis.slaveof()
- else:
- self.redis.slaveof(master, 6379)
+ try:
+ if None == master:
+ self.redis.slaveof()
+ os.system("/usr/sbin/monit unmonitor slave")
+ else:
+ self.redis.slaveof(master, 6379)
+ os.system("/usr/sbin/monit monitor slave")
+
+ os.system("/usr/sbin/monit monitor redis")
+ except Exception as e:
+ print e
if __name__ == '__main__':
# easy testing, use like this (requires environment variables)
View
4 info.sh
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-rm /var/run/redis/info
-sudo -u redis sh -c "/usr/local/bin/redis-cli info > /var/run/redis/info"
View
49 leave.py
@@ -44,20 +44,35 @@
ec2 = EC2(key, access)
if __name__ == '__main__':
- # get the host, us
- host = Host(cluster.domain.name)
- # make sure we are not connected to anything anymore
- host.set_master()
-
- node = host.get_node()
- endpoint = host.get_endpoint()
-
- # delete all there is to us
- ec2.unset_tag()
- r53_zone.delete_record(node)
- cluster.delete_node(node)
-
- # and the last to leave, please close the door
- size = cluster.size()
- if size <= 0:
- r53_zone.delete_record(cluster.domain.name)
+ try:
+ # get the host, us
+ host = Host(cluster.domain.name)
+ # make sure we are not connected to anything anymore
+ host.set_master()
+
+ node = host.get_node()
+ endpoint = host.get_endpoint()
+ except Exception as e:
+ print e
+
+ try:
+ # delete all there is to us
+ ec2.unset_tag()
+ except Exception as e:
+ print e
+
+ try:
+ r53_zone.delete_record(node)
+ except Exception as e:
+ print e
+
+ try:
+ cluster.delete_node(node)
+
+ # and the last to leave, please close the door
+ size = cluster.size()
+ print size
+ if size <= 0:
+ r53_zone.delete_record(cluster.domain.name)
+ except Exception as e:
+ print e
View
44 prepare.py
@@ -21,7 +21,7 @@
from boto.ec2.connection import EC2Connection
from boto.ec2.regioninfo import RegionInfo
-import backup
+import backup, administration
try:
url = "http://169.254.169.254/latest/"
@@ -54,6 +54,9 @@ def create_device(snapshot=None):
volume.attach(instance_id, device)
volume_id = volume.id
+ # we can't continue without a properly attached device
+ os.system("while [ ! -b {0} ] ; do /bin/true ; done".format(device))
+
# make sure the volume is deleted upon termination
# should also protect from disaster like loosing an instance
# (it doesn't work with boto, so we do it 'outside')
@@ -61,10 +64,11 @@ def create_device(snapshot=None):
# if we start from snapshot we are almost done
if snapshot == "" or None == snapshot:
+ print snapshot
# first create filesystem
os.system("/sbin/mkfs.xfs {0}".format(device))
- # mount
+ # mount, but first wait until the device is ready
os.system("/bin/mount -t xfs -o defaults {0} {1}".format(device, mount))
# and grow (if necessary)
os.system("/usr/sbin/xfs_growfs {0}".format(mount))
@@ -107,8 +111,8 @@ def prepare():
# we have a bucket, and perhaps a device. lets try to restore
# from rdb, first from metadata later from user_data.
- if "" != metadata['rdb']:
- backup.restore(key, access, cluster, metadata['rdb'])
+ if rdb != None and "" != rdb:
+ backup.restore(key, access, cluster, rdb)
latest = administration.get_latest_RDB(key, access, cluster)
if "" != latest:
@@ -119,19 +123,19 @@ def prepare():
prepare()
def meminfo():
- """
- dict of data from meminfo (str:int).
- Values are in kilobytes.
- """
- re_parser = re.compile(r'^(?P<key>\S*):\s*(?P<value>\d*)\s*kB')
- result = dict()
- for line in open('/proc/meminfo'):
- match = re_parser.match(line)
- if not match:
- continue # skip lines that don't parse
- key, value = match.groups(['key', 'value'])
- result[key] = int(value)
- return result
+ """
+ dict of data from meminfo (str:int).
+ Values are in kilobytes.
+ """
+ re_parser = re.compile(r'^(?P<key>\S*):\s*(?P<value>\d*)\s*kB')
+ result = dict()
+ for line in open('/proc/meminfo'):
+ match = re_parser.match(line)
+ if not match:
+ continue # skip lines that don't parse
+ key, value = match.groups(['key', 'value'])
+ result[key] = int(value)
+ return result
if __name__ == '__main__':
import os, sys
@@ -150,11 +154,11 @@ def meminfo():
rdb = None
# what is the domain to work with
- name = os.environ['REDIS_NAME'].strip()
- zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
+ redis_name = os.environ['REDIS_NAME'].strip()
+ hosted_zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
# the name (and identity) of the cluster (the master)
- cluster = "{0}.{1}".format(name, zone)
+ cluster = "{0}.{1}".format(redis_name, hosted_zone)
size = 3 * ( meminfo()['MemTotal'] / ( 1024 * 1024 ) )
provision(sys.argv[1], sys.argv[2], cluster, size,
View
8 route53.py
@@ -52,14 +52,16 @@ def update_record(self, name, value):
def delete_record(self, name):
changes = ResourceRecordSets(self.route53, self.zone_id)
+ value = None
sets = self.route53.get_all_rrsets(self.zone_id, None)
for rset in sets:
if rset.name == name + ".":
value = rset.resource_records[0]
- change = changes.add_change("DELETE", name + ".", "CNAME", 60)
- change.add_value(value)
- changes.commit()
+ if value != None:
+ change = changes.add_change("DELETE", name + ".", "CNAME", 60)
+ change.add_value(value)
+ changes.commit()
if __name__ == '__main__':
# easy testing, use like this (requires environment variables)
Please sign in to comment.
Something went wrong with that request. Please try again.