Merge branch 'master' of github.com:wr0ngway/rubber

lib/rubber/recipes/rubber/instances.rb

@@ -79,18 +79,22 @@
     Stop the EC2 instance for the give ALIAS
   DESC
   required_task :stop do
-    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true)
+
+    aliases = Rubber::Util::parse_aliases(instance_aliases)
     ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
-    stop_instance(instance_alias)
+    stop_instances(aliases)
   end
 
   desc <<-DESC
     Start the EC2 instance for the give ALIAS
   DESC
   required_task :start do
-    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true)
+
+    aliases = Rubber::Util::parse_aliases(instance_aliases)
     ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
-    start_instance(instance_alias)
+    start_instances(aliases)
   end
 
   desc <<-DESC
@@ -465,50 +469,106 @@ def reboot_instance(instance_alias, force=false)
 
     cloud.reboot_instance(instance_item.instance_id)
   end
-
-  # Stops the given ec2 instance.  Note that this operation only works for instances that use an EBS volume for the root
+  
+  # Stops the given ec2 instances.  Note that this operation only works for instances that use an EBS volume for the root
   # device and that are not spot instances.
-  def stop_instance(instance_alias)
-    instance_item = rubber_instances[instance_alias]
-    fatal "Instance does not exist: #{instance_alias}" if ! instance_item
-    fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil?
-    fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
-
-    env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
-
-    value = Capistrano::CLI.ui.ask("About to STOP #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
+  def stop_instances(aliases)
+    stop_threads = []
+
+    instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]}
+    instance_items = aliases.collect do |instance_alias|
+      instance_item = rubber_instances[instance_alias]
+
+      fatal "Instance does not exist: #{instance_alias}" if ! instance_item
+      fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil?
+      fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
+
+      instance_item
+    end
+
+    # Get user confirmation
+    human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ')
+    value = Capistrano::CLI.ui.ask("About to STOP #{human_instance_list} in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
     fatal("Exiting", 0) if value != "yes"
+
+    instance_items.each do |instance_item|
+      logger.info "Stopping instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}"
+
+      stop_threads << Thread.new do
+        env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
 
-    logger.info "Stopping instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
-
-    cloud.stop_instance(instance_item.instance_id)
+        cloud.stop_instance(instance_item.instance_id)
+
+        stopped = false
+        while !stopped
+          sleep 1
+          instance = cloud.describe_instances(instance_item.instance_id).first rescue {}
+          stopped = (instance[:state] == "stopped")
+        end
+      end
+    end
+
+    print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to stop"
+    while true do
+      print "."
+      sleep 2
+      break unless stop_threads.any?(&:alive?)
+    end
+    print "\n"
+
+    stop_threads.each(&:join)
   end
 
-  # Starts the given ec2 instance.  Note that this operation only works for instances that use an EBS volume for the root
+  # Starts the given ec2 instances.  Note that this operation only works for instances that use an EBS volume for the root
   # device, that are not spot instances, and that are already stopped.
-  def start_instance(instance_alias)
-    instance_item = rubber_instances[instance_alias]
-    fatal "Instance does not exist: #{instance_alias}" if ! instance_item
-    fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil?
-    fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
-
-    env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
-
-    value = Capistrano::CLI.ui.ask("About to START #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
+  def start_instances(aliases)
+    start_threads = []
+    refresh_threads = []
+
+    instance_items = aliases.collect do |instance_alias|
+      instance_item = rubber_instances[instance_alias]
+
+      fatal "Instance does not exist: #{instance_alias}" if ! instance_item
+      fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil?
+      fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
+
+      instance_item
+    end
+
+    # Get user confirmation
+    human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ')
+    value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
     fatal("Exiting", 0) if value != "yes"
-
-    logger.info "Starting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
-
-    cloud.start_instance(instance_item.instance_id)
-
-    # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values.
-    print "Waiting for instance to start"
+
+    instance_items.each do |instance_item|
+      logger.info "Starting instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}"
+
+      start_threads << Thread.new do
+        env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
+
+        cloud.start_instance(instance_item.instance_id)
+
+        # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values.
+        refresh_threads << Thread.new do
+          while ! refresh_instance(instance_item.name)
+            sleep 1
+          end
+        end
+      end
+    end
+
+    print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to start"
     while true do
       print "."
       sleep 2
-
-      break if refresh_instance(instance_alias)
+      break unless start_threads.any?(&:alive?)
     end
+
+    start_threads.each(&:join)
+    refresh_threads.each(&:join)
+
+    # Static IPs, DNS, etc. need to be set up for the started instances
+    post_refresh
   end
 
   # delete from ~/.ssh/known_hosts all lines that begin with ec2- or instance_alias

lib/rubber/recipes/rubber/static_ips.rb

@@ -19,8 +19,9 @@
           rubber_instances.save
         end
 
-        # then, associate it if we don't have a record (on instance) of association
-        if ! ic.static_ip
+        # then, associate it if we don't have a record (on instance) of association or it
+        # doesn't match the instance's current external ip
+        if !ic.static_ip || ip != ic.external_ip
           logger.info "Associating static ip #{ip} with #{ic.full_name}"
           associate_static_ip(ip, ic.instance_id)
 

templates/apache/config/rubber/rubber-apache.yml

@@ -1,6 +1,6 @@
 
 roles:
   apache:
-    packages: [apache2]
+    packages: [apache2, libapache2-mod-proxy-html, libcurl4-openssl-dev, libapache2-mod-xsendfile]
   web_tools:
-    packages: [libapache2-mod-proxy-html]
+    packages: [apache2, libapache2-mod-proxy-html, libcurl4-openssl-dev, libapache2-mod-xsendfile]

templates/base/config/rubber/rubber-ruby.yml

@@ -11,11 +11,11 @@
 packages: [build-essential, git-core, subversion, curl, autoconf, bison, ruby, zlib1g-dev, libssl-dev, libreadline6-dev, libxml2-dev, libyaml-dev]
 
 # REQUIRED: The version of ruby-build to use for building ruby.
-ruby_build_version: 20130104
+ruby_build_version: 20130118
 
 # REQUIRED: Set to the version string for the ruby version you wish to use
 # Run "ruby-build --definitions" to see the list of possible options
-ruby_version: 1.9.3-p362
+ruby_version: 1.9.3-p374
 
 # REQUIRED: Installation path for ruby.
 ruby_path: "/usr/local/rubies/#{ruby_version}"

templates/passenger/config/rubber/role/passenger/passenger-apache-vhost.conf

@@ -56,6 +56,9 @@ Listen <%= port %>
     # Cache the resource even if SSL is in use.
     Header merge Cache-Control public
 
+    # Remove any cookies set with the request so we avoid them being cached in a CDN.
+    Header unset Set-Cookie
+
     SetEnv no-gzip
   </LocationMatch>
 

templates/redis/config/rubber/rubber-redis.yml

@@ -1,4 +1,4 @@
-redis_server_version: 2.4.16
+redis_server_version: 2.4.18
 redis_server_pid_file: /var/run/redis-server.pid
 redis_server_conf_file: /etc/redis.conf
 redis_server_log_file: /var/log/redis-server.log

templates/solr/config/rubber/common/solr_sunspot.yml

@@ -0,0 +1,10 @@
+<%
+  @path = "#{Rubber.root}/config/sunspot.yml"
+  @additive =  ["#prod_start", "#prod_end"]
+%> <% rubber_instances.for_role('solr').each do |ic| %>
+production:
+    solr:
+      hostname: <%= ic.external_host %>
+      port: 8080
+      log_level: WARNING
+<% end %>

templates/solr/config/rubber/deploy-solr.rb

@@ -0,0 +1,78 @@
+# installs, starts and stops solr
+#
+# * installation is ubuntu specific
+# * start and stop tasks are using the thinking sphinx plugin
+
+namespace :rubber do
+
+  namespace :solr do
+
+    rubber.allow_optional_tasks(self)
+
+    after "rubber:install_packages", "rubber:solr:custom_install"
+
+    desc "custom installing java and solr"
+    task :custom_install, :roles => :solr do
+
+        upload rubber_env.jdk_path, "/tmp/#{rubber_env.jdk}"
+        upload rubber_env.solr_xml_path, "/tmp/#{rubber_env.solr_xml}"
+        upload rubber_env.tarz_config_files, "/tmp/solr_conf.tar.gz"
+        rubber.sudo_script 'install_java_solr', <<-ENDSCRIPT
+          if [ ! -d "/usr/lib/jvm/jdk1.7" ]; then
+            echo 'installing oracle java'
+            tar -zxf /tmp/#{rubber_env.jdk} -C /tmp
+            sudo mkdir -p /usr/lib/jvm/jdk1.7
+            mv -f /tmp/jdk1.7.0_10/* /usr/lib/jvm/jdk1.7/
+
+            echo 'updating java alterlative'
+            update-alternatives --install "/usr/bin/java" "java" "/usr/lib/jvm/jdk1.7/bin/java" 1
+            update-alternatives --install "/usr/bin/javac" "javac" "/usr/lib/jvm/jdk1.7/bin/javac" 1
+            update-alternatives --install "/usr/bin/javaws" "javaws" "/usr/lib/jvm/jdk1.7/bin/javaws" 1
+
+            echo 'installing tomcat'
+            curl -o /tmp/apache-tomcat-7.0.34.tar.gz http://ftp.heanet.ie/mirrors/www.apache.org/dist/tomcat/tomcat-7/v7.0.34/bin/apache-tomcat-7.0.34.tar.gz
+            tar -zxf /tmp/apache-tomcat-7.0.34.tar.gz -C #{rubber_env.tomcat_dest_folder}
+            rm /tmp/apache-tomcat-7.0.34.tar.gz
+
+            echo 'installing solr'
+            curl -o /tmp/apache-solr-4.0.0.tgz http://ftp.heanet.ie/mirrors/www.apache.org/dist/lucene/solr/4.0.0/apache-solr-4.0.0.tgz
+            tar -zxf /tmp/apache-solr-4.0.0.tgz -C /tmp
+            cp /tmp/apache-solr-4.0.0/dist/apache-solr-4.0.0.war /mnt/apache-tomcat-7.0.34/webapps/solr.war
+            rm -fr /tmp/apache-solr-4.0.0*
+
+            echo 'setting up solr'
+            mkdir -p #{rubber_env.solr_home_dest_foler}/solr/data
+            mkdir -p #{rubber_env.solr_home_dest_foler}/solr/#{rubber_env.core_name}
+            tar -zxf /tmp/solr_conf.tar.gz -C /mnt/solr/#{rubber_env.core_name}
+            mv /tmp/#{rubber_env.solr_xml} #{rubber_env.solr_home_dest_foler}/solr
+            rm /tmp/solr_conf.tar.gz
+          fi
+        ENDSCRIPT
+    end
+
+
+    def set_java_opts
+      "export JAVA_OPTS='-server -Xmx#{rubber_env.Xmx} -Dsolr.data.dir=#{rubber_env.solr_home_dest_foler}/solr/data -Dsolr.solr.home=#{rubber_env.solr_home_dest_foler}/solr'"
+    end
+
+    desc "start solr"
+    task :start_solr, :roles => :solr  do
+      rubber.sudo_script 'start_solr', <<-ENDSCRIPT
+        echo 'starting tomcat'
+        #{set_java_opts}
+        nohup #{rubber_env.tomcat_dest_folder}/apache-tomcat-7.0.34/bin/startup.sh  &
+        sleep 5
+      ENDSCRIPT
+    end
+
+    desc "stop solr"
+    task :stop_solr, :roles => :solr do
+      rubber.sudo_script 'stop_solr', <<-ENDSCRIPT
+        echo 'stoping tomcat'
+        #{set_java_opts}
+        #{rubber_env.tomcat_dest_folder}/apache-tomcat-7.0.34/bin/shutdown.sh
+      ENDSCRIPT
+    end
+  end
+
+end

templates/solr/config/rubber/rubber-solr.yml

@@ -0,0 +1,10 @@
+jdk: JDK_TAR_GZ_FILE_NAME
+jdk_path: JDK_TAR_GZ_FILE_NAME_INC_FILE_NAME
+solr_xml: SOLR_XML_FILE_NAME
+solr_xml_path: SOLR_XML_FILE_PATH_AND_NAME
+core_name: SOLR_CORE_NAME
+tarz_config_files: PATH_TO_SOLR_CIONFIG_TAR_GZ_FILE
+tomcat_dest_folder: TOMCAT7_DEST_FOLER
+solr_home_dest_foler: SOLR_BASE_FOLDER
+Xmx: HEAP_SPACE_FOR_JVM (e.g. 1024m or 2gb etc)
+

templates/solr/templates.yml

@@ -0,0 +1,3 @@
+description: The apache solr module
+dependent_templates:
+  - base

templates/torquebox/config/rubber/role/app/torquebox-apache-vhost.conf

@@ -1,6 +1,6 @@
 <%
-  @path = "/etc/apache2/sites-available/#{rubber_env.app_name}-passenger"
-  @post = "a2enmod rewrite && a2enmod ssl && a2enmod expires && a2enmod xsendfile && a2ensite #{rubber_env.app_name}"
+  @path = "/etc/apache2/sites-available/#{rubber_env.app_name}-torquebox"
+  @post = "a2enmod rewrite && a2enmod ssl && a2enmod expires && a2enmod xsendfile && a2ensite #{rubber_env.app_name}-torquebox"
 
   sidekiq_gem_path = if rubber_instances.for_role('sidekiq').any?
       require 'sidekiq'
@@ -27,8 +27,6 @@ Listen <%= port %>
   # Don't show haproxy checks in access log (see also apache2.conf)
   SetEnvIf Request_URI "^/httpchk.txt$" dontlog
 
-  RailsEnv  <%= Rubber.env %>
-
   XSendFile on
 
   <% if rubber_instances.for_role('sidekiq').any? %>
@@ -57,6 +55,9 @@ Listen <%= port %>
     # Cache the resource even if SSL is in use.
     Header merge Cache-Control public
 
+    # Remove any cookies set with the request so we avoid them being cached in a CDN.
+    Header unset Set-Cookie
+
     SetEnv no-gzip
   </LocationMatch>
 
@@ -79,7 +80,7 @@ Listen <%= port %>
   </FilesMatch>
   <% end %>
 
-  <% if port == rubber_env.passenger_listen_ssl_port %>
+  <% if port == rubber_env.apache_listen_ssl_port %>
   SSLEngine on
   SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire
   SSLCertificateFile    /etc/ssl/certs/ssl-cert-snakeoil.pem