From 7e4a5f289a1a3a34a2f79a8efa16fe4d5f56fe52 Mon Sep 17 00:00:00 2001 From: Emil Tin Date: Wed, 6 May 2009 21:44:43 +0200 Subject: [PATCH] capistrano integration. pp-extensions not neeed anymore for rails_deploy --- Capfile | 2 + config/deploy.rb | 66 ++++ config/poolparty/clouds.rb | 5 +- config/recipes/.svn/all-wcprops | 23 ++ config/recipes/.svn/entries | 64 ++++ config/recipes/.svn/format | 1 + .../.svn/text-base/clusters.rb.svn-base | 296 ++++++++++++++++++ config/recipes/.svn/text-base/dns.rb.svn-base | 0 .../recipes/.svn/text-base/logs.rb.svn-base | 37 +++ config/recipes/logs.rb | 37 +++ 10 files changed, 527 insertions(+), 4 deletions(-) create mode 100644 Capfile create mode 100644 config/deploy.rb create mode 100644 config/recipes/.svn/all-wcprops create mode 100644 config/recipes/.svn/entries create mode 100644 config/recipes/.svn/format create mode 100644 config/recipes/.svn/text-base/clusters.rb.svn-base create mode 100644 config/recipes/.svn/text-base/dns.rb.svn-base create mode 100644 config/recipes/.svn/text-base/logs.rb.svn-base create mode 100644 config/recipes/logs.rb diff --git a/Capfile b/Capfile new file mode 100644 index 0000000..c8dd2a0 --- /dev/null +++ b/Capfile @@ -0,0 +1,2 @@ +load 'deploy' if respond_to?(:namespace) # cap2 differentiator +load 'config/deploy' diff --git a/config/deploy.rb b/config/deploy.rb new file mode 100644 index 0000000..5f3dfc7 --- /dev/null +++ b/config/deploy.rb @@ -0,0 +1,66 @@ +# Capistrano deploy specification. +# Integrated with poolparty by getting node addresses from poolparty, and use them to set cap roles. + + +#gain access to poolparry node info +require 'poolparty' +require 'clouds.rb' + +main_cloud = :app + + +#extract the hostnames of running instances from a poolparty cloud +def find_running_nodes cloud + clouds[cloud].nodes(:status=>"running").map { |node| node.hostname } +end + +#find the rails deploy object so we can extract info +def find_rails_deploy cloud + clouds[cloud].ordered_resources.find { |node| node.is_a? RailsDeployClass } +end + +#assign cap roles to pp nodes +nodes = find_running_nodes(main_cloud) #get addresses of running instance in the poolparty cloud named 'app' +role :app, *nodes +role :web, *nodes +role :db, *nodes + +# you must have both the public and the private key in the same folder, the public key should have the extension ".pub" +# you can generate a public key from your private key by using 'ssh-keygen -y' on the command line +ssh_options[:keys] = clouds[main_cloud].keypair.filepath #use the keyfile pointed to in clouds.rb +ssh_options[:user] = "root" + + +#use info from the rails_deploy block in clouds.rb to set deploy stuff +deployer = find_rails_deploy(main_cloud) || raise("Can't find rails deploy object in clouds.rb!") +deploy_dir = "#{deployer.dsl_options[:dir]}/#{deployer.dsl_options[:name]}" +set :scm, "git" +set :application, deployer.dsl_options[:name] +set :repository, deployer.dsl_options[:repo] +set :deploy_to, deploy_dir + + +set :branch, "master" +set :deploy_via, :remote_cache +#set :user, "deployer" +#default_run_options[:pty] = true +#ssh_options[:forward_agent] = true + + +desc <<-DESC +Run whoami on the nodes. +DESC +task :stats do + run "cd #{deploy_dir}/current && RAILS_ENV=production rake stats" +end + + +desc "tail production log files" +task :logs, :roles => :app do + run "tail -f #{deploy_dir}/shared/log/production.log" do |channel, stream, data| + puts + puts "#{channel[:host]}: #{data}" +# printf '...' + break if stream == :err + end +end diff --git a/config/poolparty/clouds.rb b/config/poolparty/clouds.rb index 3458e55..8df356a 100644 --- a/config/poolparty/clouds.rb +++ b/config/poolparty/clouds.rb @@ -1,5 +1,3 @@ -require "poolparty-extensions" #you must have the auser-poolparty-extensions gem installed - pool :application do instances 1 keypair "~/.ec2/testpair" #you need to modify this to point to your EC2 key file @@ -7,13 +5,12 @@ cloud :app do has_gem_package "rails", :version => "2.3.2" #must match the version specified in the rails environment.rb has_package "mysql-client" - has_package "mysql-server" #mysql server installation + has_package "mysql-server" has_package "libmysqlclient15-dev" #so we can install the mysql gem has_gem_package "mysql" #so rails can talk to mysql has_service "mysql" #run the mysql server has_file "/etc/motd", :content => "Welcome!" #login message (message-of-the-day) - has_exec "updatedb" #make the command line 'locate' tool work has_rails_deploy "my_app" do #deploy our rails app using apache + mod_rails/passenger dir "/var/www" diff --git a/config/recipes/.svn/all-wcprops b/config/recipes/.svn/all-wcprops new file mode 100644 index 0000000..de105f4 --- /dev/null +++ b/config/recipes/.svn/all-wcprops @@ -0,0 +1,23 @@ +K 25 +svn:wc:ra_dav:version-url +V 55 +/repos/!svn/ver/1064/webapp/trunk/webapp/config/recipes +END +clusters.rb +K 25 +svn:wc:ra_dav:version-url +V 67 +/repos/!svn/ver/1064/webapp/trunk/webapp/config/recipes/clusters.rb +END +dns.rb +K 25 +svn:wc:ra_dav:version-url +V 61 +/repos/!svn/ver/918/webapp/trunk/webapp/config/recipes/dns.rb +END +logs.rb +K 25 +svn:wc:ra_dav:version-url +V 62 +/repos/!svn/ver/917/webapp/trunk/webapp/config/recipes/logs.rb +END diff --git a/config/recipes/.svn/entries b/config/recipes/.svn/entries new file mode 100644 index 0000000..446cdb6 --- /dev/null +++ b/config/recipes/.svn/entries @@ -0,0 +1,64 @@ +8 + +dir +3249 +https://dev.koblo.com:444/repos/webapp/trunk/webapp/config/recipes +https://dev.koblo.com:444/repos + + + +2008-04-11T21:16:32.233217Z +1064 +emil + + +svn:special svn:externals svn:needs-lock + + + + + + + + + + + +850aafe4-0746-43d5-bba1-ccadfad06b28 + +clusters.rb +file + + + + +2009-02-11T19:44:36.000000Z +60d00427fe9f2961937b39efc0043f5a +2008-04-11T21:16:32.233217Z +1064 +emil + +dns.rb +file + + + + +2009-02-11T19:44:36.000000Z +d41d8cd98f00b204e9800998ecf8427e +2008-03-16T11:46:45.926645Z +918 +emil + +logs.rb +file + + + + +2009-02-11T19:44:36.000000Z +30cf0dfe0e15c13be50f6f5263c57d6d +2008-03-16T10:00:59.987443Z +917 +emil + diff --git a/config/recipes/.svn/format b/config/recipes/.svn/format new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/config/recipes/.svn/format @@ -0,0 +1 @@ +8 diff --git a/config/recipes/.svn/text-base/clusters.rb.svn-base b/config/recipes/.svn/text-base/clusters.rb.svn-base new file mode 100644 index 0000000..5726eeb --- /dev/null +++ b/config/recipes/.svn/text-base/clusters.rb.svn-base @@ -0,0 +1,296 @@ + +#tasks for managing ec2 instance clusters +#image_id_32_bit is defined in the ec2onrails gem, and contains the ec2rails amazon machine id + +#TODO +#need to automatically remove machine from list of know host when killing instances, or we can get errors later + +#the database will still be called webapp_production, even when the rails environment is something else. if we want +#to change this we need to override ec2onrails cap tasks, since they assume the production name when creating the db + + + +require 'lib/ec2cluster' +require 'net/ssh' +require 'pp' + +#run a command locally, and throw an exception if there was any errors. output is the output of the command run +def locally( command ) + result = `#{command}` + raise("error: #{$?}") if ($?.to_i)>0 + result +end + +def cluster_file_path + stage = fetch(:stage ).to_s + path=File.join( locally("pwd").chomp, "tmp/clusters/#{stage}.yml") +end + + + +Capistrano::Configuration.instance(:must_exist).load do + + + + desc <<-DESC + Load ec2 server address from yml file, and setup capistrano roles. + DESC + task :after_ensure do #:ensure is part of the multistage capistrano extension, and sets the stage + cluster = EC2Cluster.new( cluster_file_path ) + if cluster.instances + [:web,:app,:db].each do |r| + a = cluster.addresses_for_role(r) + if a + a << { :primary => true } if r==:db #TODO won't work when we have more than one instance with db role + puts "[#{r.inspect}] set to #{a.inspect}" + role r, *a + make_admin_role_for r + else + puts " [#{r.inspect}] not set!" + end + end + + stage = fetch(:stage).to_s + else + puts 'Cluster file is empty.' + end + + #rails_env controls what environment to use when running db migrations + set :rails_env, stage.to_s + end + + + + namespace :deploy do + + desc <<-DESC + DESC + task :after_update_code do + upload_db_config + upload_mongrel_config + end + + desc <<-DESC + Upload a modified version of database.yml to the app servers, where host points to the primary db server. + Assumes current stage indicates the environment to modify settings for. + DESC + task :upload_db_config, :roles => :app do + config = YAML.load_file( File.join( locally("pwd").chomp, "config/database.yml") ) + c = EC2Cluster.new(cluster_file_path) + config[stage.to_s]['host'] = c.db_primary_address + put YAML.dump( config ), "#{release_path}/config/database.yml" + end + + desc <<-DESC + Upload a modified version of mongrel_cluster.yml to the app servers, using current stage as environment + DESC + task :upload_mongrel_config, :roles => :app_admin do + config = YAML.load_file( File.join( locally("pwd").chomp, "config/mongrel_cluster.yml") ) + config['environment'] = stage.to_s + pp config + c = EC2Cluster.new(cluster_file_path) + path = "/etc/mongrel_cluster/app.yml" #this path is part of the ec2onrails setup + sudo "chmod ugo+w #{path}" + put YAML.dump( config ), path + end + + + end + + + + namespace :cluster do + + + desc <<-DESC + Build, boot and launch cluster, and setup dns. + DESC + task :all_steps do + cluster.init + cluster.boot + cluster.launch + dns.update + end + + + desc <<-DESC + Build cluster file, but don't boot yet. + 2 app instances, 1 database instance. + DESC + task :init, :roles => :app do + puts 'Building cluster file....' + c = EC2Cluster.new(cluster_file_path) + c.clear + + #describe cluster by adding instances. they will not be booted yet + #FIXME you can currently only add one db instance, or roles will break because of the primary option + c.add_instance( image_id_32_bit, [:web,:app,:db], :primary => true ) + +# c.add_instance( image_id_32_bit, [:web,:app] ) +# c.add_instance( image_id_32_bit, [:web,:app] ) +# c.add_instance( image_id_32_bit, [:db], :primary => true ) + c.save + puts '--------' + c.report + puts 'OK' + end + + + desc <<-DESC + Load ec2 server address from yml file, and setup capistrano roles. + DESC + task :roles do #:ensure is part of the multistage capistrano extension, and sets the stage + #will be displayed by after_ensure + end + + desc <<-DESC + Read cluster file and show content. + DESC + task :info, :roles => :app do + cluster = EC2Cluster.new( cluster_file_path ) + puts '---------' + printf 'Loading cluster file... ' + cluster.report + puts 'OK' + end + + desc <<-DESC + Check the status off all instances. + DESC + task :check, :roles => :app do + cluster = EC2Cluster.new( cluster_file_path ) + puts '---------' + cluster.reload_info + cluster.report + puts 'OK' + end + + + desc <<-DESC + Boot cluster. Will skip already running instances. + DESC + task :boot, :roles => :app do + c = EC2Cluster.new(cluster_file_path) + c.reload_info + c.boot + cluster + end + + desc <<-DESC + Perform a cold deploy of the cluster. + DESC + task :launch do + ec2onrails.setup + ec2onrails.db.set_root_password + deploy.cold + dns.update + end + + + desc <<-DESC + Kill the cluster. Please be careful. + DESC + task :kill, :roles => :app do + raise "Sorry, but killing the production cluster is not allowed!" if stage.to_s=="production" + cluster = EC2Cluster.new(cluster_file_path) + cluster.kill + cluster.delete + end + + + + desc <<-DESC + Add an server to the cluster with roles [app web]. + DESC + task :add_app_instance, :roles => :app do + c = EC2Cluster.new(cluster_file_path) + i = EC2Instance.new(image_id_32_bit,[:web,:app] ) + c.add( i ) + i.boot + i.await_running + c.await_network + c.reload_info + c.save + + cluster + puts "Instance #{i.address} added." + puts "To launch instance, use: cap launch HOSTS=#{i.address}" + puts "To remove instance, use: cap remove_instance HOSTS=#{i.address}" + puts "OK" + end + + desc <<-DESC + Remove an instance from cluster. Use must specify the address using HOSTS=
+ DESC + task :remove_instance, :roles => :app do + raise 'You must specify servers using HOST=
' unless ENV['HOSTS'] + c = EC2Cluster.new(cluster_file_path) + c.remove_instance( ENV['HOSTS'] ) + cluster + end + + desc <<-DESC + Find ip address of web servers + DESC + task :web_ips, :roles => :web do + #simple extract ip from amazon address name - FIXME is robust? can we safely assume the ip matches the address name? + c = EC2Cluster.new(cluster_file_path) + puts c.ips_for_role( :web ) + + #alternative 1 - query aws info server + #-s is silent mode. 169.254.169.254 is an aws server which can be queried for info about the instance sending the request + # result = [] + # run "curl -s http://169.254.169.254/latest/meta-data/public-ipv4" do |channel, stream, data| + # result << {channel[:host] => data } + # break if stream == :err + # end + # for r in result + # r.each { |k,v| puts "#{v} (#{k})" } + # end + + #alternative 2 - run arp locally + # c = EC2Cluster.new(cluster_file_path) + # ips = c.ips_for_role(:web) + # puts "Public IP adddresses for web servers: #{ips.inspect}" + end + + + end + + namespace :dns do + + desc <<-DESC + Upload dns.yml to the web servers, containing list of address/updatekey pairs + DESC + task :upload_config, :roles => :web do + c = EC2Cluster.new(cluster_file_path) + config = c.get_dns_config + put YAML.dump( config ), "#{current_path}/config/dns.yml" + end + + desc <<-DESC + Update dynanic DNS A-entries at freedns.afraid.org. + Each server contacts the dns server and passes key that identifies the entry to be updated. + The entry will then be set to point to the ip of the server requsting the update. + This means each server must send a different key. The way we do it is to first upload the file dns.yml, + which contains a list of address/key pairs. We then run the update_dns script on all the web servers, + which will look up the key according to the servers own address. + DESC + task :update, :roles => :web do + freedns_keys = YAML.load_file( File.join( locally("pwd").chomp, "config/freedns.yml") ) + c = EC2Cluster.new(cluster_file_path) + printf 'Assigning freedns keys to web servers...' + c.assign_dns freedns_keys[stage.to_s] + puts 'ok' + c.save + upload_config + + run "cd #{current_path}" + run "#{current_path}/script/update_dns" + puts "DNS server updated. Note that changes might take a while to spread across the net." + end + + end + + +end \ No newline at end of file diff --git a/config/recipes/.svn/text-base/dns.rb.svn-base b/config/recipes/.svn/text-base/dns.rb.svn-base new file mode 100644 index 0000000..e69de29 diff --git a/config/recipes/.svn/text-base/logs.rb.svn-base b/config/recipes/.svn/text-base/logs.rb.svn-base new file mode 100644 index 0000000..3143fc5 --- /dev/null +++ b/config/recipes/.svn/text-base/logs.rb.svn-base @@ -0,0 +1,37 @@ +Capistrano::Configuration.instance(:must_exist).load do + + #these two log file tasks are based on http://errtheblog.com/posts/19-streaming-capistrano + #they are modified to handle different deployment stages + + desc "tail production log files" + task :tail_logs, :roles => :app do + run "tail -f #{shared_path}/log/#{stage}.log" do |channel, stream, data| + puts + puts "#{channel[:host]}: #{data}" + # printf '...' + break if stream == :err + end + end + + desc "check production log files in textmate(tm)" + task :mate_logs, :roles => :app do + require 'tempfile' + tmp = Tempfile.open('w') + logs = Hash.new { |h,k| h[k] = '' } + + run "tail -n 500 #{shared_path}/log/#{stage}.log" do |channel, stream, data| + logs[channel[:host]] << data + break if stream == :err + end + + logs.each do |host, log| + tmp.write("--- #{host} ---\n\n") + tmp.write(log + "\n") + end + + tmp.flush + `mate -w -l 99999999 #{tmp.path}` #place cursor at some huge line nr to go to end of file + tmp.close + end + +end \ No newline at end of file diff --git a/config/recipes/logs.rb b/config/recipes/logs.rb new file mode 100644 index 0000000..3143fc5 --- /dev/null +++ b/config/recipes/logs.rb @@ -0,0 +1,37 @@ +Capistrano::Configuration.instance(:must_exist).load do + + #these two log file tasks are based on http://errtheblog.com/posts/19-streaming-capistrano + #they are modified to handle different deployment stages + + desc "tail production log files" + task :tail_logs, :roles => :app do + run "tail -f #{shared_path}/log/#{stage}.log" do |channel, stream, data| + puts + puts "#{channel[:host]}: #{data}" + # printf '...' + break if stream == :err + end + end + + desc "check production log files in textmate(tm)" + task :mate_logs, :roles => :app do + require 'tempfile' + tmp = Tempfile.open('w') + logs = Hash.new { |h,k| h[k] = '' } + + run "tail -n 500 #{shared_path}/log/#{stage}.log" do |channel, stream, data| + logs[channel[:host]] << data + break if stream == :err + end + + logs.each do |host, log| + tmp.write("--- #{host} ---\n\n") + tmp.write(log + "\n") + end + + tmp.flush + `mate -w -l 99999999 #{tmp.path}` #place cursor at some huge line nr to go to end of file + tmp.close + end + +end \ No newline at end of file