Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

whole lot of refactoring, intermediate better-safe-than-sorry commit

  • Loading branch information...
commit 4760ea6b0cc6e1a6cd8146333209518fea4868c2 1 parent b62fab9
@truthtrap truthtrap authored
View
4 backup.py
@@ -78,8 +78,8 @@ def create_bucket(key, access, cluster):
# our create_bucket is not idempotent, we can't recreate buckets
try:
s3.create_bucket( cluster.replace('.', '-'), location=Location.EU)
- except S3CreateError:
- pass
+ except S3CreateError as e:
+ print e
def put_RDB(key, access, cluster, prefix='hourly'):
s3 = S3Connection(key, access)
View
6 backup.sh
@@ -17,5 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
-source /root/config.sh
-python backup.py $1 ${EC2_KEY_ID} ${EC2_SECRET_KEY} $2 $3
+dirname=`dirname $0`
+
+source ${dirname}/config.sh
+python ${dirname}/backup.py $1 ${EC2_KEY_ID} ${EC2_SECRET_KEY} $2 $3
View
26 cluster.py
@@ -47,8 +47,8 @@ def __init__(self, key, access, cluster):
url = "http://169.254.169.254/latest/meta-data/"
public_hostname = urlopen(url + "public-hostname").read()
- availability_zone = urlopen(url + "placement/availability-zone").read()
- region = availability_zone[:-1]
+ zone = urlopen(url + "placement/availability-zone").read()
+ region = zone[:-1]
except:
sys.exit("We should be getting user-data here...")
@@ -62,6 +62,12 @@ def __init__(self, key, access, cluster):
self.domain = sdb.create_domain(cluster)
self.metadata = self.domain.get_item('metadata', True)
+ if None == self.metadata:
+ self.metadata = self.domain.new_item('metadata')
+
+ self.metadata.add_value('master', '')
+ self.metadata.add_value('slave', '')
+ self.metadata.save()
def add_node(self, node, endpoint):
try:
@@ -140,15 +146,23 @@ def get_endpoint(self, node):
def get_master(self, node=None):
try:
- return self.domain.get_item(node)['master']
+ master = self.domain.get_item(node)['master']
+
+ return master
except:
- return None
+ return node
def get_slave(self, node=None):
try:
- return self.domain.get_item(node)['slave']
+ slave = self.domain.get_item(node)['slave']
+
+ return slave
except:
- return None
+ return node
+
+ def size(self):
+ select = "select count(*) from `{0}` where itemName() like '%.{0}'".format(self.domain.name)
+ return int(self.domain.select(select).next()['Count'])
def check_integrity(self, cluster):
pass
View
2  config.example.sh 100644 → 100755
@@ -69,7 +69,7 @@ export EC2_REGION=${EC2_AVAILABILITY_ZONE:0:${#EC2_AVAILABILITY_ZONE}-1}
export EC2_INSTANCE_ID=$($curl $instance_data_url/meta-data/instance-id)
# this will only work with a patched simpledb (see SIMPLEDB)
-export SDB_SERVICE_URL='https://sdb.${EC2_REGION}.amazonaws.com'
+export SDB_SERVICE_URL="https://sdb.${EC2_REGION}.amazonaws.com"
# changing this is entirely your own responsibility, I wouldn't do it
export SQS_TASK_QUEUE="${REDIS_NAME}-tasks"
View
23 cron.d/high.cron
@@ -15,19 +15,22 @@
# You should have received a copy of the GNU General Public License
# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+# Do not remove 'INSTALLPATH', it will be used when setting up a new
+# Redis server.
+
# m h dom mon dow command
-@hourly /root/backup.sh rdb hourly > /dev/null 2>&1
-15 */2 * * * /root/backup.sh snapshot hourly > /dev/null 2>&1
+@hourly INSTALLPATH/backup.sh rdb hourly > /dev/null 2>&1
+15 */2 * * * INSTALLPATH/backup.sh snapshot hourly > /dev/null 2>&1
-@daily /root/backup.sh rdb daily > /dev/null 2>&1
-@daily /root/backup.sh snapshot daily > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh rdb daily > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh snapshot daily > /dev/null 2>&1
-@weekly /root/backup.sh rdb weekly > /dev/null 2>&1
-@weekly /root/backup.sh snapshot weekly > /dev/null 2>&1
+@weekly INSTALLPATH/backup.sh rdb weekly > /dev/null 2>&1
+@weekly INSTALLPATH/backup.sh snapshot weekly > /dev/null 2>&1
-@monthly /root/backup.sh rdb monthly > /dev/null 2>&1
-@monthly /root/backup.sh snapshot monthly > /dev/null 2>&1
+@monthly INSTALLPATH/backup.sh rdb monthly > /dev/null 2>&1
+@monthly INSTALLPATH/backup.sh snapshot monthly > /dev/null 2>&1
-@daily /root/backup.sh purge > /dev/null 2>&1
-@daily /root/backup.sh purge > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh purge > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh purge > /dev/null 2>&1
View
11 cron.d/low.cron
@@ -15,9 +15,12 @@
# You should have received a copy of the GNU General Public License
# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+# Do not remove 'INSTALLPATH', it will be used when setting up a new
+# Redis server.
+
# m h dom mon dow command
-@hourly /root/backup.sh rdb hourly > /dev/null 2>&1
-@daily /root/backup.sh rdb daily > /dev/null 2>&1
-@weekly /root/backup.sh rdb weekly > /dev/null 2>&1
-@monthly /root/backup.sh rdb monthly > /dev/null 2>&1
+@hourly INSTALLPATH/backup.sh rdb hourly > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh rdb daily > /dev/null 2>&1
+@weekly INSTALLPATH/backup.sh rdb weekly > /dev/null 2>&1
+@monthly INSTALLPATH/backup.sh rdb monthly > /dev/null 2>&1
View
23 cron.d/normal.cron
@@ -15,19 +15,22 @@
# You should have received a copy of the GNU General Public License
# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+# Do not remove 'INSTALLPATH', it will be used when setting up a new
+# Redis server.
+
# m h dom mon dow command
-@hourly /root/backup.sh rdb hourly > /dev/null 2>&1
-15 */2 * * * /root/backup.sh snapshot hourly > /dev/null 2>&1
+@hourly INSTALLPATH/backup.sh rdb hourly > /dev/null 2>&1
+15 */2 * * * INSTALLPATH/backup.sh snapshot hourly > /dev/null 2>&1
-@daily /root/backup.sh rdb daily > /dev/null 2>&1
-@daily /root/backup.sh snapshot daily > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh rdb daily > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh snapshot daily > /dev/null 2>&1
-@weekly /root/backup.sh rdb weekly > /dev/null 2>&1
-@weekly /root/backup.sh snapshot weekly > /dev/null 2>&1
+@weekly INSTALLPATH/backup.sh rdb weekly > /dev/null 2>&1
+@weekly INSTALLPATH/backup.sh snapshot weekly > /dev/null 2>&1
-@monthly /root/backup.sh rdb monthly > /dev/null 2>&1
-@monthly /root/backup.sh snapshot monthly > /dev/null 2>&1
+@monthly INSTALLPATH/backup.sh rdb monthly > /dev/null 2>&1
+@monthly INSTALLPATH/backup.sh snapshot monthly > /dev/null 2>&1
-@daily /root/backup.sh purge > /dev/null 2>&1
-@daily /root/backup.sh purge > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh purge > /dev/null 2>&1
+@daily INSTALLPATH/backup.sh purge > /dev/null 2>&1
View
71 decommission.py
@@ -0,0 +1,71 @@
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+import os, sys
+import json, urllib2
+
+from boto.ec2.connection import EC2Connection
+from boto.ec2.regioninfo import RegionInfo
+
+import administration, backup
+
+try:
+ url = "http://169.254.169.254/latest/"
+
+ userdata = json.load(urllib2.urlopen(url + "user-data"))
+except Exception as e:
+ print e
+ exit( "We couldn't get user-data or other meta-data...")
+
+device = "/dev/sdf"
+mount = "/var/lib/redis"
+
+def decommission(key, access, cluster, persistence="no"):
+ # make a last backup
+ if "no" != persistence:
+ # take the latest RDB and move it to S3
+ rdb = backup.put_RDB(key, access, cluster, 'monthly')
+ administration.set_RDB(key, access, cluster, rdb)
+
+ # if we have a volume, make a last snapshot
+ if "low" != persistence:
+ snapshot = backup.make_snapshot(key, access, cluster, 'monthly')
+ administration.add_snapshot(key, access, cluster, snapshot)
+
+ # we don't have to get rid any the volume, it is deleted on termination
+
+ # change to the default (no persistence)
+ os.system("/bin/rm -f /etc/redis/redis.conf")
+ # and empty the cron as well
+ os.system("/bin/echo | /usr/bin/crontab")
+
+if __name__ == '__main__':
+ import os, sys
+
+ try:
+ persistence = userdata['persistence']
+ except:
+ persistence = None
+
+ # what is the domain to work with
+ name = os.environ['REDIS_NAME'].strip()
+ zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
+
+ # the name (and identity) of the cluster (the master)
+ cluster = "{0}.{1}".format(name, zone)
+
+ decommission(sys.argv[1], sys.argv[2], cluster, persistence=persistence)
View
26 decommission.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+# pretty sad, but this is the easiest way to share
+# the configuration
+
+dirname=`dirname $0`
+
+source ${dirname}/config.sh
+python ${dirname}/decommission.py ${EC2_KEY_ID} ${EC2_SECRET_KEY}
View
30 etc/monit/redis.monitrc
@@ -0,0 +1,30 @@
+ # add this to the main monit with 'include <path>/redis.monitrc'
+
+ # INFO gives a bit more than 955 bytes
+ set expectbuffer 2 kb
+
+ # always check the local redis
+ check process redis with pidfile /var/run/redis/redis.pid
+ start program = "/etc/init.d/redis start"
+ stop program = "/etc/init.d/redis stop"
+ if failed port 6379 then restart
+ if 2 restarts within 3 cycles then timeout
+ group redis
+
+ # a redis can have 2 states: master or slave. we want to check
+ # for health of these 2 different states. we'll be using these
+ # checks, turning them on and off.
+
+ # and check if local redis thinks it has a healthy master
+ check host is-slave with address localhost
+ if failed port 6379
+ send "INFO\r\n" expect ".*role:slave.*master_link_status:up.*"
+ then exec "/root/ReDiS/remaster.sh"
+ group redis
+
+ # or check if local redis is a master
+ check host is-master with address localhost
+ if failed port 6379
+ send "INFO\r\n" expect ".*role:master.*"
+ then exec "/root/ReDiS/remaster.sh"
+ group redis
View
73 host.py
@@ -0,0 +1,73 @@
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+
+import os, sys
+
+import json
+import hashlib
+import redis
+
+from urllib2 import urlopen
+
+#
+# REDIS HOST
+#
+# ...
+#
+class Host:
+ def __init__(self, cluster):
+ try:
+ url = "http://169.254.169.254/latest/"
+ self.endpoint = urlopen(url + "meta-data/public-hostname").read()
+ self.userdata = json.load(urlopen(url + "user-data"))
+ except:
+ sys.exit("We should be getting user-data here...")
+
+ self.cluster = cluster
+ self.id = hashlib.md5(self.endpoint).hexdigest()[:8]
+ self.node = "{0}.{1}".format(self.id, self.cluster)
+
+ self.redis = redis.StrictRedis(host="localhost", port=6379)
+
+ def get_node(self):
+ return self.node
+
+ def get_endpoint(self):
+ return self.endpoint
+
+ def get_master(self):
+ return self.master
+
+ def set_master(self, master=None):
+ self.master = master
+ if None == master:
+ self.redis.slaveof()
+
+ os.system("/usr/sbin/monit unmonitor is-slave")
+ os.system("/usr/sbin/monit monitor is-master")
+ else:
+ self.redis.slaveof(master, 6379)
+
+ os.system("/usr/sbin/monit monitor is-slave")
+ os.system("/usr/sbin/monit unmonitor is-master")
+
+if __name__ == '__main__':
+ # easy testing, use like this (requires environment variables)
+ # python host.py set_master cluster 2c922342a.cluster
+ host = Host(sys.argv[2])
+ print getattr(host, sys.argv[1])(*sys.argv[3:])
View
2  identity.py
@@ -78,7 +78,7 @@ def set_endpoint(key, access, cluster):
# ok, add ourselves as master (head)
change = changes.add_change("CREATE", name, "CNAME", 60)
change.add_value(fqdn)
- except:
+ else:
print "Couldn't add Route53 record or set the instance Name tag for " + name + ". It probably already exists."
changes.commit()
View
58 join.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import json
+
+from urllib2 import urlopen
+
+from cluster import Cluster
+from host import Host
+from route53 import Route53Zone
+
+# your amazon keys
+key = os.environ['EC2_KEY_ID']
+access = os.environ['EC2_SECRET_KEY']
+
+# what is the domain to work with
+name = os.environ['REDIS_NAME'].strip()
+zone_name = os.environ['HOSTED_ZONE_NAME']
+zone_id = os.environ['HOSTED_ZONE_ID']
+
+# the name (and identity) of the cluster (the master)
+cluster = "{0}.{1}".format(name, zone_name.rstrip('.'))
+
+# get/create the cluster environment
+cluster = Cluster(key, access, cluster)
+r53_zone = Route53Zone(key, access, zone_id)
+
+if __name__ == '__main__':
+ # and get the instance up and running
+ host = Host(cluster.domain.name)
+
+ node = host.get_node()
+ endpoint = host.get_endpoint()
+
+ # now we are ready to be (added to) the cluster
+ cluster.add_node(node, endpoint)
+ r53_zone.create_record(node, endpoint)
+
+ master = cluster.get_master(node)
+ # if we don't have a master, we ARE the master
+ if master == node:
+ r53_zone.update_record(cluster.domain.name, endpoint)
View
7 launch.sh → join.sh
@@ -19,5 +19,8 @@
# pretty sad, but this is the easiest way to share
# the configuration
-source /root/config.sh
-python /root/launch.py
+
+dirname=`dirname $0`
+
+source ${dirname}/config.sh
+python ${dirname}/join.py
View
58 leave.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import json
+
+from urllib2 import urlopen
+
+from cluster import Cluster
+from host import Host
+from route53 import Route53Zone
+
+# your amazon keys
+key = os.environ['EC2_KEY_ID']
+access = os.environ['EC2_SECRET_KEY']
+
+# what is the domain to work with
+name = os.environ['REDIS_NAME'].strip()
+zone_name = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
+zone_id = os.environ['HOSTED_ZONE_ID']
+
+# the name (and identity) of the cluster (the master)
+cluster = "{0}.{1}".format(name, zone_name)
+
+# get/create the cluster environment
+cluster = Cluster(key, access, cluster)
+r53_zone = Route53Zone(key, access, zone_id)
+
+if __name__ == '__main__':
+ # and get the instance up and running
+ host = Host(cluster.domain.name)
+
+ node = host.get_node()
+ endpoint = host.get_endpoint()
+
+ # now we are ready to be (added to) the cluster
+ cluster.delete_node(node)
+ r53_zone.delete_record(node)
+
+ # if we are the last, we need cleanup a bit
+ size = cluster.size()
+ if size <= 0:
+ r53_zone.delete_record(cluster.domain.name)
View
26 leave.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+# pretty sad, but this is the easiest way to share
+# the configuration
+
+dirname=`dirname $0`
+
+source ${dirname}/config.sh
+python ${dirname}/leave.py
View
161 prepare.py
@@ -0,0 +1,161 @@
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+import os, sys, re
+import json, urllib2
+
+from boto.ec2.connection import EC2Connection
+from boto.ec2.regioninfo import RegionInfo
+
+import backup
+
+try:
+ url = "http://169.254.169.254/latest/"
+
+ userdata = json.load(urllib2.urlopen(url + "user-data"))
+ instance_id = urllib2.urlopen(url + "meta-data/instance-id").read()
+
+ zone = urllib2.urlopen(url + "meta-data/placement/availability-zone").read()
+ region = zone[:-1]
+except Exception as e:
+ print e
+ exit( "We couldn't get user-data or other meta-data...")
+
+device = "/dev/sdf"
+mount = "/var/lib/redis"
+
+def provision(key, access, cluster, size, persistence="no", snapshot=None, rdb=None):
+ # ec2 is region specific
+ region_info = RegionInfo(name=region,
+ endpoint="ec2.{0}.amazonaws.com".format(region))
+ ec2 = EC2Connection(key, access, region=region_info)
+
+ def create_device(snapshot=None):
+ # if we have the device (/dev/sdf) just don't do anything anymore
+ mapping = ec2.get_instance_attribute(instance_id, 'blockDeviceMapping')
+ try:
+ volume_id = mapping['blockDeviceMapping'][device].volume_id
+ except:
+ volume = ec2.create_volume(size, zone, snapshot)
+ volume.attach(instance_id, device)
+ volume_id = volume.id
+
+ # make sure the volume is deleted upon termination
+ # should also protect from disaster like loosing an instance
+ # (it doesn't work with boto, so we do it 'outside')
+ os.system("/usr/bin/ec2-modify-instance-attribute --block-device-mapping \"{0}=:true\" {1} --region {2}".format(device, instance_id, region))
+
+ # if we start from snapshot we are almost done
+ if snapshot == "" or None == snapshot:
+ # first create filesystem
+ os.system("/sbin/mkfs.xfs {0}".format(device))
+
+ # mount
+ os.system("/bin/mount -t xfs -o defaults {0} {1}".format(device, mount))
+ # and grow (if necessary)
+ os.system("/usr/sbin/xfs_growfs {0}".format(mount))
+
+ return volume_id
+
+ def prepare():
+ # from this point we are sure we don't have to be careful
+ # with local files/devices/disks/etc
+
+ # we are going to work with local files, we need our path
+ path = os.path.dirname(os.path.abspath(__file__))
+
+ dst = "/etc/redis/redis.conf"
+ redis = "{0}/etc/redis/{1}.conf".format(path, persistence)
+ cron = "{0}/cron.d/{1}.cron".format(path, persistence)
+
+ # redis will start with this conf
+ os.system("/bin/ln -fs {0} {1}".format(redis, dst))
+ # and root's cron will be set accordingly as well
+ os.system("/bin/sed 's:INSTALLPATH:{0}:' {1} | /usr/bin/crontab".format(path, cron))
+
+ # ok, ready to set up assets like bucket and volume
+ if "no" != persistence:
+ backup.create_bucket(key, access, cluster)
+
+ try:
+ # only try to create one if we have one
+ if "" == snapshot or None == snapshot:
+ raise Exception('metadata','empty snapshot')
+ else:
+ create_device(snapshot)
+ except:
+ try:
+ latest = administration.get_latest_snapshot(key,
+ access, cluster)
+ create_device(latest)
+ except:
+ create_device()
+
+ # we have a bucket, and perhaps a device. lets try to restore
+ # from rdb, first from metadata later from user_data.
+ if "" != metadata['rdb']:
+ backup.restore(key, access, cluster, metadata['rdb'])
+
+ latest = administration.get_latest_RDB(key, access, cluster)
+ if "" != latest:
+ backup.restore(key, access, cluster, latest)
+
+ # if we have a valid mount, we don't do anything
+ if os.path.ismount(mount) == False:
+ prepare()
+
+def meminfo():
+ """
+ dict of data from meminfo (str:int).
+ Values are in kilobytes.
+ """
+ re_parser = re.compile(r'^(?P<key>\S*):\s*(?P<value>\d*)\s*kB')
+ result = dict()
+ for line in open('/proc/meminfo'):
+ match = re_parser.match(line)
+ if not match:
+ continue # skip lines that don't parse
+ key, value = match.groups(['key', 'value'])
+ result[key] = int(value)
+ return result
+
+if __name__ == '__main__':
+ import os, sys
+
+ try:
+ persistence = userdata['persistence']
+ except:
+ persistence = None
+ try:
+ snapshot = userdata['snapshot']
+ except:
+ snapshot = None
+ try:
+ rdb = userdata['rdb']
+ except:
+ rdb = None
+
+ # what is the domain to work with
+ name = os.environ['REDIS_NAME'].strip()
+ zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
+
+ # the name (and identity) of the cluster (the master)
+ cluster = "{0}.{1}".format(name, zone)
+ size = 3 * ( meminfo()['MemTotal'] / ( 1024 * 1024 ) )
+
+ provision(sys.argv[1], sys.argv[2], cluster, size,
+ persistence=persistence, snapshot=snapshot, rdb=rdb)
View
26 prepare.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+# pretty sad, but this is the easiest way to share
+# the configuration
+
+dirname=`dirname $0`
+
+source ${dirname}/config.sh
+python ${dirname}/prepare.py ${EC2_KEY_ID} ${EC2_SECRET_KEY}
View
37 launch.py → remaster.py
@@ -17,24 +17,39 @@
import os
import sys
+import json
-import identity, administration, setup
+from urllib2 import urlopen
+
+from cluster import Cluster
+from host import Host
# your amazon keys
key = os.environ['EC2_KEY_ID']
access = os.environ['EC2_SECRET_KEY']
+# what is the domain to work with
+name = os.environ['REDIS_NAME'].strip()
+zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
+
+# the name (and identity) of the cluster (the master)
+name = "{0}.{1}".format(name, zone)
+
+# get/create the cluster environment
+cluster = Cluster(key, access, name, userdata)
+
if __name__ == '__main__':
- # what is the domain to work with
- name = os.environ['REDIS_NAME'].strip()
- zone = os.environ['HOSTED_ZONE_NAME'].rstrip('.')
- cluster = "{0}.{1}".format(name, zone)
+ # first, we have to get the instance
+ host = Host(key, access, name, userdata)
- # make sure we persist the cluster meta data
- administration.set_cluster_metadata(key, access, cluster)
+ node = host.get_node()
+ endpoint = host.get_endpoint()
- # now, set the endpoints in route53 and as Name tag
- identity.set_endpoint(key, access, cluster)
+ # get the failing master
+ old = cluster.get_master(node)
+ cluster.delete_node(old)
- # now provision the instance
- setup.provision(key, access, cluster)
+ new = cluster.get_master(node)
+ host.set_master(new)
+ if new == None:
+ route53.update_record(name, endpoint)
View
26 remaster.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+# pretty sad, but this is the easiest way to share
+# the configuration
+
+dirname=`dirname $0`
+
+source ${dirname}/config.sh
+python ${dirname}/remaster.py
View
68 route53.py
@@ -0,0 +1,68 @@
+# Copyright (C) 2011, 2012 9apps B.V.
+#
+# This file is part of Redis for AWS.
+#
+# Redis for AWS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Redis for AWS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
+
+import os, sys
+import platform
+import json, urllib2
+
+from boto.route53.connection import Route53Connection
+from boto.route53.record import ResourceRecordSets
+
+class Route53Zone:
+ def __init__(self, key, access, zone_id):
+ self.zone_id = zone_id
+ self.route53 = Route53Connection(key, access)
+
+ def create_record(self, name, value):
+ changes = ResourceRecordSets(self.route53, self.zone_id)
+
+ change = changes.add_change("CREATE", name + ".", "CNAME", 60)
+ change.add_value(value)
+ changes.commit()
+
+ def update_record(self, name, value):
+ changes = ResourceRecordSets(self.route53, self.zone_id)
+
+ sets = self.route53.get_all_rrsets(self.zone_id, None)
+ for rset in sets:
+ if rset.name == name + ".":
+ previous_value = rset.resource_records[0]
+
+ change = changes.add_change("DELETE", name + ".", "CNAME", 60)
+ change.add_value(previous_value)
+
+ change = changes.add_change("CREATE", name + ".", "CNAME", 60)
+ change.add_value(value)
+ changes.commit()
+
+ def delete_record(self, name):
+ changes = ResourceRecordSets(self.route53, self.zone_id)
+
+ sets = self.route53.get_all_rrsets(self.zone_id, None)
+ for rset in sets:
+ if rset.name == name + ".":
+ value = rset.resource_records[0]
+
+ change = changes.add_change("DELETE", name + ".", "CNAME", 60)
+ change.add_value(value)
+ changes.commit()
+
+if __name__ == '__main__':
+ # easy testing, use like this (requires environment variables)
+ # python route53.py create_record key access id name value
+ r53_zone = Route53Zone(sys.argv[2], sys.argv[3], sys.argv[4])
+ print getattr(r53_zone, sys.argv[1])(*sys.argv[5:])
View
2  setup.py
@@ -89,7 +89,7 @@ def prepare(persistence):
# redis will start with this conf
os.system("/bin/ln -fs {0} {1}".format(redis, dst))
# and root's cron will be set accordingly as well
- os.system("/usr/bin/crontab {0}".format(cron))
+ os.system("/bin/sed 's/INSTALLPATH/{0}/' {1} | /usr/bin/crontab".format(path, cron))
# ok, ready to set up assets like bucket and volume
if "no" != persistence:
View
6 terminate.sh
@@ -17,5 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with Redis for AWS. If not, see <http://www.gnu.org/licenses/>.
-source /root/config.sh
-python /root/terminate.py ${REDIS_NAME}
+dirname=`dirname $0`
+
+source ${dirname}/config.sh
+python ${dirname}/terminate.py ${REDIS_NAME}
Please sign in to comment.
Something went wrong with that request. Please try again.