Permalink
Browse files

first commit

  • Loading branch information...
0 parents commit 596a5c1b011dc1f39bef3c5d358b422b005be097 mig5 committed Nov 27, 2010
Showing with 1,036 additions and 0 deletions.
  1. +674 −0 LICENSE.txt
  2. +15 −0 README
  3. +25 −0 TODO
  4. +3 −0 config/backup_restore_creds
  5. +7 −0 config/felicity.ini
  6. +106 −0 felicity
  7. +16 −0 get_limits
  8. +18 −0 print_servers
  9. +16 −0 scripts/backup_list_buckets
  10. +53 −0 scripts/backup_restore
  11. +31 −0 scripts/backup_restore_wrapper
  12. +72 −0 scripts/firewall
@@ -0,0 +1,15 @@
+Dodgy little python thing that:
+
+ * Deploys a new server at Rackspace Cloud (by way of libcloud)
+
+ * Installs a bunch of stuff like duplicity, python-boto, some scripts
+
+ * Sets up a firewall, configures ssh to reject password auth
+
+ * Runs a restore of backups from the Amazon S3 cloud based buckets found that match arg(1) (server name)
+
+ * Runs a collection-status on the backups too
+
+ * emails the results to arg(2) (e-mail address)
+
+Tonnes of ugly hacks at this stage, working on things from the TODO.
@@ -0,0 +1,25 @@
+Things to do
+
+ * Improve reading in of configs, no having to hardcode the connections/drivers
+
+ * Improve the efficiency of scripts, not sure if we really need to use `at` but it seemed to solve the issue of duplicity simply not running?
+
+ * Improve the deployment commands (MultiStepDeployment doesn't seem to like taking an array inside an array)
+
+ * There is no error checking right now
+
+ * Ability to choose only certain buckets and not just all for that server
+
+ * Perhaps pythonise the whole thing, no shell scripts
+
+ * allow SSH in from only the machine issuing the command?
+
+ * make things more configurable in .ini
+
+ * perhaps don't deploy on RackSpace cloud. Seems every now and then RS has a bug where networking doesn't work until reboot of the guest
+
+ * make a Drupal module that executes this and takes Provider/User/API key from a form (security issues?)
+
+ * work out a way to **actually** self destruct and not just halt
+
+ * some sort of webdav or something so people don't have to ssh in unless they want to? hook in ldap auth for this? (support system)
@@ -0,0 +1,3 @@
+export AWS_ACCESS_KEY_ID=your_key_id
+export AWS_SECRET_ACCESS_KEY=your_secret_key
+export PASSPHRASE=your_gpg_passphrase_for_duplicity
@@ -0,0 +1,7 @@
+[Rackspace]
+user=your_username
+key=your_api_key
+distro=Debian
+[Amazon]
+user=your_user_key_id
+key=your_secret_api_key
@@ -0,0 +1,106 @@
+#! /usr/bin/env python
+
+from libcloud.types import Provider
+from libcloud.providers import get_driver
+from libcloud.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
+from libcloud.ssh import SSHClient, ParamikoSSHClient
+from random import choice
+import paramiko
+from paramiko.rsakey import RSAKey
+import os
+import random
+import sys
+import ConfigParser
+
+hostname = 'backup-restore-"%s"%d' % tuple([sys.argv[1], random.randrange(0, 101, 2)])
+ssh = 0
+restorer_passwd = 0
+
+def ssh_connect(remote_host):
+ rsa_key = os.path.expanduser('~/.ssh/id_rsa')
+ mykey = paramiko.RSAKey.from_private_key_file(rsa_key)
+ global ssh
+ ssh = paramiko.SSHClient()
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ ssh.load_system_host_keys()
+ print "Connecting via SSH to %s" % remote_host
+ ssh.connect(remote_host, username='root', pkey=mykey)
+
+def ssh_close():
+ print "Closing SSH connection"
+ ssh.close()
+
+def run_script():
+ print "Preparing restore script to run"
+ ssh.exec_command('echo "/usr/local/bin/backup_restore_wrapper %s %s" | at now + 1 minute' % tuple([ sys.argv[1], sys.argv[2]]))
+ print "Setting self-destruct for 48 hours"
+ ssh.exec_command('echo "halt" | at now + 2 days')
+ print "Disabling password authentication in SSH"
+ ssh.exec_command('sed -i -r -e "s/^[ #]*(PasswordAuthentication).*/\1 no/" /etc/ssh/sshd_config')
+ ssh.exec_command('/etc/init.d/ssh restart')
+ print "Setting a firewall"
+ ssh.exec_command('update-rc.d /etc/init.d/firewall defaults')
+ ssh.exec_command('/etc/init.d/firewall start')
+
+def deploy_script():
+ ftp = ssh.open_sftp()
+ print "Uploading scripts"
+ ftp.put(os.path.expanduser('scripts/backup_restore_wrapper'), '/usr/local/bin/backup_restore_wrapper')
+ ftp.put(os.path.expanduser('scripts/backup_restore'), '/usr/local/bin/backup_restore')
+ ftp.put(os.path.expanduser('scripts/backup_list_buckets'), '/usr/local/bin/backup_list_buckets')
+ ftp.put(os.path.expanduser('config/backup_restore_creds'), '/usr/local/etc/backup_restore_creds')
+ ftp.put(os.path.expanduser('scripts/firewall'), '/etc/init.d/firewall')
+
+ scripts = ['/usr/local/bin/backup_restore_wrapper', '/usr/local/bin/backup_restore', '/usr/local/bin/backup_list_buckets', '/etc/init.d/firewall']
+ for script in scripts:
+ script_file = ftp.open(script, 'r')
+ script_file.chmod(0755)
+ script_file.close()
+ ftp.close()
+
+def main():
+
+ config = ConfigParser.RawConfigParser()
+ config.read('config/felicity.ini')
+ user = config.get('Rackspace', 'user')
+ key = config.get('Rackspace', 'key')
+ distro = config.get('Rackspace', 'distro')
+
+ Driver = get_driver(Provider.RACKSPACE)
+ conn = Driver(user, key)
+
+ images = conn.list_images()
+ sizes = conn.list_sizes()
+
+ dist_image = [image for image in images if distro in image.name]
+ assert len(dist_image) == 1, "We found more than one image for %s, will be assuming the first one" % distro
+
+ cmd1 = SSHKeyDeployment(open(os.path.expanduser("~/.ssh/id_rsa.pub")).read())
+ cmd2 = ScriptDeployment("echo 'deb http://www.backports.org/debian lenny-backports main contrib non-free' >> /etc/apt/sources.list")
+ cmd3 = ScriptDeployment("apt-get update")
+ cmd4 = ScriptDeployment("apt-get -y -t lenny-backports install duplicity")
+ cmd5 = ScriptDeployment("apt-get -y install python-boto")
+ cmd6 = ScriptDeployment("echo 'postfix postfix/main_mailer_type select Internet Site' | debconf-set-selections")
+ cmd7 = ScriptDeployment("echo 'postfix postfix/mailname string $HOSTNAME' | debconf-set-selections")
+ cmd8 = ScriptDeployment("echo 'postfix postfix/destinations string localhost.localdomain, localhost' | debconf-set-selections")
+ cmd9 = ScriptDeployment("apt-get -y install postfix mailx")
+ cmd10 = ScriptDeployment("apt-get install at")
+ msd = MultiStepDeployment([cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7, cmd8, cmd9, cmd10])
+
+ # deploy_node takes the same base keyword arguments as create_node.
+ print "Provisioning server and running deployment processes"
+ node = conn.deploy_node(name=hostname, image=dist_image[0], size=sizes[0], deploy=msd)
+
+ remote_host = node.public_ip[0]
+
+ ssh_connect(remote_host)
+
+ deploy_script()
+
+ run_script()
+
+ ssh_close()
+
+
+if __name__ == "__main__":
+ main()
@@ -0,0 +1,16 @@
+#! /usr/bin/env python
+
+from libcloud.types import Provider
+from libcloud.providers import get_driver
+import ConfigParser
+
+config = ConfigParser.RawConfigParser()
+config.read('config/felicity.ini')
+user = config.get('Rackspace', 'user')
+key = config.get('Rackspace', 'key')
+
+Driver = get_driver(Provider.RACKSPACE)
+conn = Driver(user, key)
+
+
+print conn.ex_limits()
@@ -0,0 +1,18 @@
+#! /usr/bin/env python
+
+from libcloud.types import Provider
+from libcloud.providers import get_driver
+import ConfigParser
+
+config = ConfigParser.RawConfigParser()
+config.read('config/felicity.ini')
+user = config.get('Rackspace', 'user')
+key = config.get('Rackspace', 'key')
+
+Driver = get_driver(Provider.RACKSPACE)
+conn = Driver(user, key)
+
+images = conn.list_images()
+
+for image in images:
+ print(image)
@@ -0,0 +1,16 @@
+#! /usr/bin/env python
+
+from boto.s3.connection import S3Connection
+import sys
+import re
+
+server = sys.argv[1]
+
+conn = S3Connection()
+
+rs = conn.get_all_buckets()
+
+for b in rs:
+ matchObj = re.match(server, b.name)
+ if matchObj:
+ print b.name
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Backup restore script
+#
+# Copyright (C) 2010 Miguel Jacq
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+source /usr/local/etc/backup_restore_creds
+
+if [ -z "$1" ]; then
+ echo "No server name provided."
+ exit 1
+fi
+
+shopt -s nullglob
+
+IP=`/sbin/ifconfig eth0 | grep "inet addr" | awk '{print $2}' | cut -d: -f2`
+SERVER=$1
+
+# Restore directory
+RESTORE_DIR=/tmp/backups/$SERVER-`date '+%F-%H%M'`
+
+BUCKETS=`python /usr/local/bin/backup_list_buckets $SERVER`
+
+# Greeting text for the email report
+echo "This is a backup restore of $SERVER executed on `date '+%F'` at `date '+%H:%M'`."
+echo "The backups will be restored to $RESTORE_DIR on a temporary server with the IP $IP"
+echo "This machine will self destruct after 48 hours."
+echo "----------------------------------------------------------------------------"
+
+# Loop over the backup directories and do a restore of each one to
+# a subdirectory of $RESTORE_DIR
+for bucket in ${BUCKETS[*]}; do
+ echo "Creating $RESTORE_DIR/$bucket"
+ mkdir -p $RESTORE_DIR/$bucket
+ echo "Restoring $SERVER bucket $bucket to $RESTORE_DIR/$bucket"
+ duplicity restore s3+http://$bucket $RESTORE_DIR/$bucket
+ echo "Now running a collection-status report of your backups in $bucket"
+ duplicity collection-status s3+http://$bucket
+ echo "----------------------------------------------------------------------------"
+done
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Source the creds
+source /usr/local/etc/backup_restore_creds
+
+SERVER=$1
+EMAIL=$2
+
+# Run the main script
+sh /usr/local/bin/backup_restore $SERVER > $HOME/emailmessage.txt
+
+unset PASSPHRASE
+
+# Cleanup
+sed -i '/^Copying duplicity/d' $HOME/emailmessage.txt
+sed -i '/^Deleting local/d' $HOME/emailmessage.txt
+sed -i '/^Local and Remote metadata/d' $HOME/emailmessage.txt
+sed -i '/^Archive dir/d' $HOME/emailmessage.txt
+sed -i '/^Connecting with backend/d' $HOME/emailmessage.txt
+sed -i '/^Synchronizing remote metadata to local cache/d' $HOME/emailmessage.txt
+
+# Email the report
+SUBJECT="Backup restore of $SERVER"
+
+mail -s "$SUBJECT" "$EMAIL" < $HOME/emailmessage.txt
+
+# Remove the email log
+rm $HOME/emailmessage.txt
+
+# Remove the duplicity cache
+rm -rf $HOME/.cache/duplicity
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# IP interfaces
+eth0=`ifconfig eth0 | grep "inet addr" | awk '{print $2}' | cut -d: -f2`
+
+# Change to 1 to enable logging of dropped packets
+LOG=0
+
+flush() {
+ iptables --flush
+ iptables --delete-chain
+}
+
+start() {
+
+ # Flush just in case 'start' has been called twice without stop or restart
+ flush
+
+ # Default policies
+ iptables --policy INPUT DROP
+ iptables --policy OUTPUT DROP
+ iptables --policy FORWARD DROP
+
+ # Accept all on loopback
+ iptables -A INPUT -i lo -j ACCEPT
+ iptables -A OUTPUT -o lo -j ACCEPT
+
+ # Accept all packets that are part of an established connection
+ iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
+ iptables -A OUTPUT -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT
+
+ # Pings
+ iptables -A INPUT -p icmp --icmp-type 8 -s 0/0 -d $eth0 -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT
+ iptables -A OUTPUT -p icmp --icmp-type 0 -s $eth0 -d 0/0 -m state --state ESTABLISHED,RELATED -j ACCEPT
+
+ # SSH
+ iptables -I INPUT -d $eth0 -p tcp --dport 22 -j ACCEPT
+
+ if [ $LOG -eq 1 ]; then
+ # Create a LOGDROP chain to log and drop packets
+ iptables -N LOGDROP
+ iptables -A LOGDROP -j LOG
+ iptables -A LOGDROP -j DROP
+ # Drop and log all other traffic inbound
+ iptables -A INPUT -j LOGDROP
+ else
+ # Drop all other traffic inbound
+ iptables -A INPUT -j DROP
+ fi
+}
+
+stop() {
+ flush
+ iptables --policy INPUT ACCEPT
+ iptables --policy OUTPUT ACCEPT
+ iptables --policy FORWARD ACCEPT
+}
+
+case "$1" in
+ start)
+ start
+ ;;
+ stop)
+ stop
+ ;;
+ restart)
+ stop
+ start
+ ;;
+ *)
+ ;;
+esac

0 comments on commit 596a5c1

Please sign in to comment.