Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

New rubber config, minimal

  • Loading branch information...
commit 3855a5a12407284d8628ae8de676b0d2d73e0842 1 parent 148ef06
Karl Brightman karlbright authored
Showing with 3,961 additions and 3 deletions.
  1. +3 −3 config/deploy.rb
  2. +18 −0 config/rubber/common/crontab
  3. +12 −0 config/rubber/common/database.yml
  4. +7 −0 config/rubber/common/ntp-sysctl.conf
  5. +57 −0 config/rubber/common/ntp.conf
  6. +71 −0 config/rubber/common/rsyslog.conf
  7. +13 −0 config/rubber/common/rubber.profile
  8. +59 −0 config/rubber/deploy-apache.rb
  9. +48 −0 config/rubber/deploy-collectd.rb
  10. +255 −0 config/rubber/deploy-graphite.rb
  11. +33 −0 config/rubber/deploy-monit.rb
  12. +45 −0 config/rubber/deploy-passenger.rb
  13. +140 −0 config/rubber/deploy-postgresql.rb
  14. +107 −0 config/rubber/deploy-setup.rb
  15. +91 −0 config/rubber/deploy-util.rb
  16. +10 −0 config/rubber/role/apache/deflate.conf
  17. +9 −0 config/rubber/role/apache/expires.conf
  18. +6 −0 config/rubber/role/apache/headers.conf
  19. +7 −0 config/rubber/role/apache/monit-apache.conf
  20. +8 −0 config/rubber/role/apache/ports.conf
  21. +52 −0 config/rubber/role/apache/setenvif.conf
  22. +21 −0 config/rubber/role/collectd/collectd-ping.conf
  23. +69 −0 config/rubber/role/collectd/collectd.conf
  24. +35 −0 config/rubber/role/collectd/filters.conf
  25. +21 −0 config/rubber/role/collectd/graphite-collectd.conf
  26. +88 −0 config/rubber/role/collectd/thresholds.conf
  27. +5 −0 config/rubber/role/collectd/types.db
  28. +232 −0 config/rubber/role/graphite_server/carbon.conf
  29. +16 −0 config/rubber/role/graphite_server/graphite_server-upstart.conf
  30. +19 −0 config/rubber/role/graphite_server/storage-schemas.conf
  31. +8 −0 config/rubber/role/graphite_web/crontab
  32. +60 −0 config/rubber/role/graphite_web/dashboard.conf
  33. +385 −0 config/rubber/role/graphite_web/dashboard.html
  34. +62 −0 config/rubber/role/graphite_web/graphite-vhost.conf
  35. +10 −0 config/rubber/role/graphite_web/graphite.wsgi
  36. +98 −0 config/rubber/role/graphite_web/local_settings.py
  37. +18 −0 config/rubber/role/monit/monit-default.conf
  38. +8 −0 config/rubber/role/monit/monit-postfix.conf
  39. +252 −0 config/rubber/role/monit/monit.conf
  40. +7 −0 config/rubber/role/passenger/collectd-passenger.conf
  41. +77 −0 config/rubber/role/passenger/collectd-passenger.rb.conf
  42. +90 −0 config/rubber/role/passenger/passenger-apache-vhost.conf
  43. +8 −0 config/rubber/role/passenger/passenger-status-sudoers.conf
  44. +45 −0 config/rubber/role/passenger/passenger.conf
  45. +22 −0 config/rubber/role/postgresql/collectd-postgresql.conf
  46. +16 −0 config/rubber/role/postgresql/crontab
  47. +10 −0 config/rubber/role/postgresql/monit-postgresql.conf
  48. +98 −0 config/rubber/role/postgresql/pg_hba.conf
  49. +7 −0 config/rubber/role/postgresql/postgresql-sysctl.conf
  50. +574 −0 config/rubber/role/postgresql/postgresql.conf
  51. +23 −0 config/rubber/role/postgresql_slave/collectd-postgresql-threshold.conf
  52. +135 −0 config/rubber/role/postgresql_slave/recovery.conf
  53. +11 −0 config/rubber/role/web_tools/monit-admin-nginx.conf
  54. +19 −0 config/rubber/role/web_tools/rubber-web-tools-monit.html
  55. +122 −0 config/rubber/role/web_tools/tools-apache-vhost.conf
  56. +7 −0 config/rubber/role/web_tools/tools-apache.auth
  57. +38 −0 config/rubber/role/web_tools/tools-index.html
  58. +6 −0 config/rubber/rubber-apache.yml
  59. +11 −0 config/rubber/rubber-collectd.yml
  60. +7 −0 config/rubber/rubber-dns.yml
  61. +22 −0 config/rubber/rubber-graphite.yml
  62. +11 −0 config/rubber/rubber-monit.yml
  63. +16 −0 config/rubber/rubber-passenger.yml
  64. +45 −0 config/rubber/rubber-postgresql.yml
  65. +42 −0 config/rubber/rubber-rvm.yml
  66. +34 −0 config/rubber/rubber.yml
6 config/deploy.rb
View
@@ -11,9 +11,9 @@
# Use a simple directory tree copy here to make demo easier.
# You probably want to use your own repository for a real app
-set :scm, :git
-set :repository, "git://github.com/Whatlock/doyle.git"
-set :deploy_via, :remote_cache
+set :scm, :none
+set :repository, "."
+set :deploy_via, :copy
# Easier to do system level config as root - probably should do it through
# sudo in the future. We use ssh keys for access, so no passwd needed
18 config/rubber/common/crontab
View
@@ -0,0 +1,18 @@
+<%
+ @read_cmd = 'crontab -l'
+ @write_cmd = 'crontab -'
+
+%>
+
+# cron clears out environment variables, but Rubber.root/script/rubber uses
+# "rvm do default" to run, so no longer any need to setup ruby env vars here,
+# all we need is PATH
+PATH=<%= ENV['PATH'] %>
+
+MAILTO=<%= rubber_env.admin_email %>
+# don't need to set RUBBER_ENV for each script since we set it for all cron here
+RUBBER_ENV=<%= Rubber.env %>
+RAILS_ENV=<%= Rubber.env %>
+
+# Roll all rails logs at midnight
+0 0 * * * <%= Rubber.root %>/script/rubber cron --task util:rotate_logs --directory=<%= Rubber.root %>/log
12 config/rubber/common/database.yml
View
@@ -0,0 +1,12 @@
+<%
+ @path = "#{Rubber.root}/config/database.yml"
+%>
+
+<%= Rubber.env %>:
+ adapter: postgresql
+ encoding: unicode
+ database: <%= rubber_env.db_name %>
+ username: <%= rubber_env.db_user %>
+ password: <%= rubber_env.db_pass %>
+ host: <%= rubber_instances.for_role('db', 'primary' => true).first.full_name %>
+ pool: 5
7 config/rubber/common/ntp-sysctl.conf
View
@@ -0,0 +1,7 @@
+<%
+ @path = "/etc/sysctl.d/60-ntp.conf"
+ @post = "/usr/sbin/service procps start"
+%>
+
+# allows us to set machine clock (xen clocks from ec2 tend to drift a lot)
+xen.independent_wallclock=1
57 config/rubber/common/ntp.conf
View
@@ -0,0 +1,57 @@
+<%
+ @path = "/etc/ntp.conf"
+ @post = "/usr/sbin/service ntp restart"
+%>
+# /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help
+
+driftfile /var/lib/ntp/ntp.drift
+
+
+# Enable this if you want statistics to be logged.
+#statsdir /var/log/ntpstats/
+
+statistics loopstats peerstats clockstats
+filegen loopstats file loopstats type day enable
+filegen peerstats file peerstats type day enable
+filegen clockstats file clockstats type day enable
+
+
+# You do need to talk to an NTP server or two (or three).
+server 0.us.pool.ntp.org
+server 1.us.pool.ntp.org
+server 2.us.pool.ntp.org
+server 3.us.pool.ntp.org
+
+<% rubber_instances.each do |ic| %>
+peer <%= ic.full_name %>
+<% end %>
+
+# Access control configuration; see /usr/share/doc/ntp-doc/html/accopt.html for
+# details. The web page <http://support.ntp.org/bin/view/Support/AccessRestrictions>
+# might also be helpful.
+#
+# Note that "restrict" applies to both servers and clients, so a configuration
+# that might be intended to block requests from certain clients could also end
+# up blocking replies from your own upstream servers.
+
+# By default, exchange time with everybody, but don't allow configuration.
+restrict -4 default kod notrap nomodify nopeer noquery
+restrict -6 default kod notrap nomodify nopeer noquery
+
+# Local users may interrogate the ntp server more closely.
+restrict 127.0.0.1
+restrict ::1
+
+# Clients from this (example!) subnet have unlimited access, but only if
+# cryptographically authenticated.
+#restrict 192.168.123.0 mask 255.255.255.0 notrust
+
+
+# If you want to provide time to your local subnet, change the next line.
+# (Again, the address is an example only.)
+#broadcast 192.168.123.255
+
+# If you want to listen to time broadcasts on your local subnet, de-comment the
+# next lines. Please do this only if you trust everybody on the network!
+#disable auth
+#broadcastclient
71 config/rubber/common/rsyslog.conf
View
@@ -0,0 +1,71 @@
+<%
+ log_dir = "/var/log/#{rubber_env.app_name}"
+ log_spool_dir = "/var/spool/rsyslog"
+
+ @path = "/etc/rsyslog.conf"
+ @post = <<-ENDSCRIPT
+ mkdir -p #{log_dir} #{log_spool_dir}
+ chown syslog:adm #{log_dir} #{log_spool_dir}
+ /usr/sbin/service rsyslog restart
+ ENDSCRIPT
+%>
+
+# /etc/rsyslog.conf Configuration file for rsyslog.
+#
+# For more information see
+# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
+#
+# Default logging rules can be found in /etc/rsyslog.d/50-default.conf
+
+
+#################
+#### MODULES ####
+#################
+
+$ModLoad imuxsock # provides support for local system logging
+$ModLoad imklog # provides kernel logging support (previously done by rklogd)
+#$ModLoad immark # provides --MARK-- message capability
+
+$KLogPath /proc/kmsg
+
+# provides UDP syslog reception
+$ModLoad imudp
+$UDPServerRun 514
+
+# provides TCP syslog reception
+#$ModLoad imtcp
+#$InputTCPServerRun 514
+
+$WorkDirectory <%= log_spool_dir %>
+$template LogCollectorDynFile,"<%= log_dir %>/%$YEAR%/%$MONTH%/%$DAY%/%programname%.log"
+
+###########################
+#### GLOBAL DIRECTIVES ####
+###########################
+
+#
+# Use traditional timestamp format.
+# To enable high precision timestamps, comment out the following line.
+#
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+# Don't filter duplicated messages
+$RepeatedMsgReduction off
+
+#
+# Set the default permissions for all log files.
+#
+$FileOwner syslog
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+$PrivDropToUser syslog
+$PrivDropToGroup adm
+# https://bugs.launchpad.net/ubuntu/+source/rsyslog/+bug/484336
+
+#
+# Include all config files in /etc/rsyslog.d/
+#
+$IncludeConfig /etc/rsyslog.d/*.conf
+
13 config/rubber/common/rubber.profile
View
@@ -0,0 +1,13 @@
+<%
+ @path = "/etc/profile.d/rubber.sh"
+ current_path = "/mnt/#{rubber_env.app_name}-#{Rubber.env}/current"
+%>
+
+# convenience to simply running rails console, etc with correct env
+export RUBBER_ENV=<%= Rubber.env %>
+export RAILS_ENV=<%= Rubber.env %>
+alias current="cd <%= current_path %>"
+alias release="cd <%= Rubber.root %>"
+
+# Always use rubygems
+export RUBYOPT="rubygems"
59 config/rubber/deploy-apache.rb
View
@@ -0,0 +1,59 @@
+
+namespace :rubber do
+
+ namespace :apache do
+
+ rubber.allow_optional_tasks(self)
+
+ after "rubber:install_packages", "rubber:apache:install"
+
+ task :install, :roles => :apache do
+ rubber.sudo_script 'install_apache', <<-ENDSCRIPT
+ a2dissite default
+
+ # TODO: remove this once 12.04 is fixed
+ # https://bugs.launchpad.net/ubuntu/+source/mod-proxy-html/+bug/964397
+ if [[ ! -f /usr/lib/libxml2.so.2 ]]; then
+ ln -sf /usr/lib/x86_64-linux-gnu/libxml2.so.2 /usr/lib/libxml2.so.2
+ fi
+ ENDSCRIPT
+ end
+
+ # serial_task can only be called after roles defined - not normally a problem, but
+ # rubber auto-roles don't get defined till after all tasks are defined
+ on :load do
+ rubber.serial_task self, :serial_restart, :roles => :apache do
+ rsudo "service apache2 stop; service apache2 start"
+ end
+ rubber.serial_task self, :serial_reload, :roles => :apache do
+ rsudo "if ! ps ax | grep -v grep | grep -c apache2 &> /dev/null; then service apache2 start; else service apache2 reload; fi"
+ end
+ end
+
+ before "deploy:stop", "rubber:apache:stop"
+ after "deploy:start", "rubber:apache:start"
+ after "deploy:restart", "rubber:apache:reload"
+
+ desc "Stops the apache web server"
+ task :stop, :roles => :apache do
+ rsudo "service apache2 stop || true"
+ end
+
+ desc "Starts the apache web server"
+ task :start, :roles => :apache do
+ rsudo "service apache2 start"
+ end
+
+ desc "Restarts the apache web server"
+ task :restart, :roles => :apache do
+ serial_restart
+ end
+
+ desc "Reloads the apache web server"
+ task :reload, :roles => :apache do
+ serial_reload
+ end
+
+ end
+
+end
48 config/rubber/deploy-collectd.rb
View
@@ -0,0 +1,48 @@
+
+namespace :rubber do
+
+ namespace :collectd do
+
+ rubber.allow_optional_tasks(self)
+
+ after "rubber:bootstrap", "rubber:collectd:bootstrap"
+ after "deploy:restart", "rubber:collectd:restart_rubber_plugin"
+
+
+ task :bootstrap, :roles => :collectd do
+ exists = capture("echo $(grep Rubber /etc/collectd/collectd.conf 2> /dev/null)")
+ if exists.strip.size == 0
+ rubber.update_code_for_bootstrap
+
+ rubber.run_config(:file => "role/collectd", :force => true, :deploy_path => release_path)
+
+ restart
+ end
+ end
+
+ desc "Start collectd system monitoring"
+ task :start, :roles => :collectd do
+ rsudo "service collectd start"
+ end
+
+ desc "Stop collectd system monitoring"
+ task :stop, :roles => :collectd do
+ rsudo "service collectd stop || true"
+ end
+
+ desc "Restart collectd system monitoring"
+ task :restart, :roles => :collectd do
+ stop
+ start
+ end
+
+ desc "Restart collectd rubber plugin"
+ task :restart_rubber_plugin, :roles => :collectd do
+ # Need to kill rubber collectd runner script to force collectd to restart
+ # it after deploy so that the runner script gets the new paths
+ rsudo "pkill -fn #{rubber_env.rubber_collectd_runner.sub(/./, '[\0]')} ; exit 0"
+ end
+
+ end
+
+end
255 config/rubber/deploy-graphite.rb
View
@@ -0,0 +1,255 @@
+
+namespace :rubber do
+
+ namespace :graphite do
+
+ rubber.allow_optional_tasks(self)
+
+ after "rubber:install_packages", "rubber:graphite:install_collectd_graphite_plugin"
+
+ task :install_collectd_graphite_plugin, :roles => :collectd do
+ rubber.sudo_script 'install_collectd', <<-ENDSCRIPT
+ if [[ ! -f "/usr/local/share/perl/5.10.1/Collectd/Plugins/Graphite.pm" ]]; then
+ wget --no-check-certificate -qNO /tmp/joemiller-collectd-graphite.tar.gz https://github.com/joemiller/collectd-graphite/tarball/#{rubber_env.collectd_graphite_plugin_version}
+ tar -C /tmp -zxf /tmp/joemiller-collectd-graphite.tar.gz
+ cd /tmp/joemiller-collectd-graphite-*
+ perl Makefile.PL
+ make
+ make test
+ make install
+ fi
+ ENDSCRIPT
+ end
+
+ # graphite collectd plugin needs newer collectd (>= 4.9)
+ before "rubber:install_packages", "rubber:graphite:setup_apt_backport_of_collectd"
+
+ task :setup_apt_backport_of_collectd do
+ sources = <<-EOF
+ deb http://gb.archive.ubuntu.com/ubuntu/ maverick main universe
+ deb-src http://gb.archive.ubuntu.com/ubuntu/ maverick main universe
+ EOF
+ sources.gsub!(/^[ \t]*/, '')
+
+ prefs = <<-EOF
+ Package: *
+ Pin: release a=maverick
+ Pin-Priority: 400
+
+ Package: libxml2
+ Pin: release a=maverick
+ Pin-Priority: 900
+
+ Package: libxml2-dev
+ Pin: release a=maverick
+ Pin-Priority: 900
+
+ Package: collectd-core
+ Pin: release a=maverick
+ Pin-Priority: 900
+
+ Package: collectd-utils
+ Pin: release a=maverick
+ Pin-Priority: 900
+
+ Package: collectd
+ Pin: release a=maverick
+ Pin-Priority: 900
+ EOF
+ prefs.gsub!(/^[ \t]*/, '')
+
+ rubber.sudo_script 'setup_apt_backport_of_collectd', <<-ENDSCRIPT
+ release=`lsb_release -sr`
+ needs_backport=`echo "$release <= 10.04" | bc`
+ if [[ $needs_backport == 1 && ! -f /etc/apt/preferences.d/rubber-collectd ]]; then
+ echo -e #{sources.inspect} > /etc/apt/sources.list.d/rubber-collectd.list
+ echo -e #{prefs.inspect} > /etc/apt/preferences.d/rubber-collectd
+ fi
+ ENDSCRIPT
+ end
+
+ task :install_graphite_from_repo, :roles => [:graphite_server, :graphite_web] do
+ rubber.sudo_script 'install_graphite_from_repo', <<-ENDSCRIPT
+ if [[ ! -d "/opt/graphite" ]]; then
+ mkdir -p /tmp/graphite_install
+ cd /tmp/graphite_install
+ bzr branch lp:graphite -r #{rubber_env.graphite_repo_version}
+
+ pushd graphite/whisper
+ python setup.py install
+ popd
+
+ pushd graphite/carbon
+ python setup.py install
+ popd
+
+ pushd graphite
+ python setup.py install
+ popd
+ fi
+ ENDSCRIPT
+ end
+
+ desc <<-DESC
+ Cleans out old whisper storage files for non-existant instances so they don't show in webui
+ DESC
+ task :clean_storage, :roles => [:graphite_server] do
+ active_instances = rubber_instances.collect {|ic| ic.name }.sort
+ stored_instances = capture("ls /opt/graphite/storage/whisper/servers/").split.sort
+ cleaning_instances = stored_instances - active_instances
+
+ if cleaning_instances.size > 0
+ logger.info "Cleaning dead instances from graphite storage: #{cleaning_instances.join(',')}"
+
+ do_clean = true
+ if (cleaning_instances.size.to_f / stored_instances.size) > 0.01
+ value = Capistrano::CLI.ui.ask("Graphite storage files to be cleaned exceeds threshold, proceed? [y/N]?: ")
+ do_clean = (value =~ /^y/)
+ end
+
+ if do_clean
+ rsudo "rm -rf /opt/graphite/storage/whisper/servers/{#{cleaning_instances.join(',')}}"
+ end
+
+ end
+ end
+
+ namespace :server do
+
+ rubber.allow_optional_tasks(self)
+
+ if Rubber::Configuration.rubber_env.graphite_repo_version
+ after "rubber:install_packages", "rubber:graphite:install_graphite_from_repo"
+ else
+ after "rubber:install_packages", "rubber:graphite:server:install"
+ end
+
+ after "rubber:bootstrap", "rubber:graphite:server:bootstrap"
+
+ desc <<-DESC
+ Installs graphite server components
+ DESC
+ task :install, :roles => :graphite_server do
+ rubber.sudo_script 'install_graphite_server', <<-ENDSCRIPT
+ if [[ ! -f "/opt/graphite/bin/carbon-cache.py" ]]; then
+ wget -qNP /tmp #{rubber_env.graphite_whisper_package_url}
+ tar -C /tmp -zxf /tmp/#{rubber_env.graphite_whisper_package_url.gsub(/.*\//, '')}
+ cd /tmp/#{rubber_env.graphite_whisper_package_url.gsub(/.*\//, '').gsub('.tar.gz', '')}
+ python setup.py install
+
+ wget -qNP /tmp #{rubber_env.graphite_carbon_package_url}
+ tar -C /tmp -zxf /tmp/#{rubber_env.graphite_carbon_package_url.gsub(/.*\//, '')}
+ cd /tmp/#{rubber_env.graphite_carbon_package_url.gsub(/.*\//, '').gsub('.tar.gz', '')}
+ python setup.py install
+
+ fi
+ ENDSCRIPT
+ end
+
+ task :bootstrap, :roles => :graphite_server do
+ exists = capture("echo $(ls /opt/graphite/storage/whisper/ 2> /dev/null)")
+ if exists.strip.size == 0
+ rubber.update_code_for_bootstrap
+
+ rubber.run_config(:file => "role/graphite_server", :force => true, :deploy_path => release_path)
+
+ restart
+ end
+ end
+
+ desc "Start graphite system monitoring"
+ task :start, :roles => :graphite_server do
+ rsudo "service graphite-server start"
+ end
+
+ desc "Stop graphite system monitoring"
+ task :stop, :roles => :graphite_server do
+ rsudo "service graphite-server stop || true"
+ end
+
+ desc "Restart graphite system monitoring"
+ task :restart, :roles => :graphite_server do
+ stop
+ start
+ end
+ end
+
+ namespace :web do
+
+ rubber.allow_optional_tasks(self)
+
+ if Rubber::Configuration.rubber_env.graphite_repo_version
+ after "rubber:graphite:server:install", "rubber:graphite:install_graphite_from_repo"
+ else
+ after "rubber:graphite:server:install", "rubber:graphite:web:install"
+ end
+
+ after "rubber:graphite:server:bootstrap", "rubber:graphite:web:bootstrap"
+
+ desc <<-DESC
+ Installs graphite web components
+ DESC
+ task :install, :roles => :graphite_web do
+ rubber.sudo_script 'install_graphite_web', <<-ENDSCRIPT
+ if [[ ! -d "/opt/graphite/webapp" ]]; then
+ wget -qNP /tmp #{rubber_env.graphite_web_package_url}
+ tar -C /tmp -zxf /tmp/#{rubber_env.graphite_web_package_url.gsub(/.*\//, '')}
+ cd /tmp/#{rubber_env.graphite_web_package_url.gsub(/.*\//, '').gsub('.tar.gz', '')}
+ # python check-dependencies.py
+ python setup.py install
+ fi
+ ENDSCRIPT
+ end
+
+ task :bootstrap, :roles => :graphite_web do
+ exists = capture("echo $(ls /opt/graphite/storage/graphite.db 2> /dev/null)")
+ if exists.strip.size == 0
+ rubber.update_code_for_bootstrap
+
+ rubber.run_config(:file => "role/graphite_web", :force => true, :deploy_path => release_path)
+
+ # django email validation barfs on localhost, but for full_host to work
+ # in admin_email, we need a env with host defined
+ web_instance = rubber_instances.for_role("graphite_web").first
+ env = rubber_cfg.environment.bind("graphite_web", web_instance.name)
+ email = env.admin_email
+
+ rubber.sudo_script 'bootstrap_graphite_web', <<-ENDSCRIPT
+ cd /opt/graphite/webapp/graphite
+ python manage.py syncdb --noinput
+ python manage.py createsuperuser --username admin --email #{email} --noinput
+ python manage.py shell <<EOF
+from django.contrib.auth.models import User
+u = User.objects.get(username__exact='admin')
+u.set_password('admin1')
+u.save()
+EOF
+ mkdir -p /opt/graphite/storage/log/webapp/
+ chown -R www-data:www-data /opt/graphite/storage/
+ ENDSCRIPT
+
+ restart
+ end
+ end
+
+ desc "Start graphite system monitoring"
+ task :start, :roles => :graphite_web do
+ rsudo "service apache2 start"
+ end
+
+ desc "Stop graphite system monitoring"
+ task :stop, :roles => :graphite_web do
+ rsudo "service apache2 stop || true"
+ end
+
+ desc "Restart graphite system monitoring"
+ task :restart, :roles => :graphite_web do
+ stop
+ start
+ end
+
+ end
+
+ end
+
+end
33 config/rubber/deploy-monit.rb
View
@@ -0,0 +1,33 @@
+
+namespace :rubber do
+
+ namespace :monit do
+
+ rubber.allow_optional_tasks(self)
+
+ # monit needs to get stopped first and started last so that it doesn't
+ # mess with us restarting everything as part of a deploy.
+ before "rubber:pre_stop", "rubber:monit:stop"
+ before "rubber:pre_restart", "rubber:monit:stop"
+ after "rubber:post_start", "rubber:monit:start"
+ after "rubber:post_restart", "rubber:monit:start"
+
+ desc "Start monit daemon monitoring"
+ task :start, :roles => :monit do
+ rsudo "service monit start"
+ end
+
+ desc "Stop monit daemon monitoring"
+ task :stop, :roles => :monit do
+ rsudo "service monit stop || true"
+ end
+
+ desc "Restart monit daemon monitoring"
+ task :restart, :roles => :monit do
+ stop
+ start
+ end
+
+ end
+
+end
45 config/rubber/deploy-passenger.rb
View
@@ -0,0 +1,45 @@
+
+namespace :rubber do
+
+ namespace :passenger do
+
+ rubber.allow_optional_tasks(self)
+
+ after "rubber:install_gems", "rubber:passenger:custom_install"
+
+ task :custom_install, :roles => :passenger do
+ rubber.sudo_script 'install_passenger', <<-ENDSCRIPT
+ # can't use passenger_lib from rubber here as it only evaluates correctly
+ # when variable interpolation of rvm_gem_home is run remotely, and since we
+ # are in cap, we run the interpolation locally
+ #
+ passenger_lib=$(find /usr/local/rvm/gems/`rvm current` -path "*/passenger-#{rubber_env.passenger_version}/*/mod_passenger.so" 2> /dev/null)
+ if [[ -z $passenger_lib ]]; then
+ echo -en "\n\n\n\n" | passenger-install-apache2-module
+ rvm #{rubber_env.rvm_ruby} --passenger
+ fi
+ ENDSCRIPT
+ end
+
+ # passenger does things differently for rack apps, so if this is a rails app, remove config.ru
+ # to stop passenger from treating this like a rack app
+ after "deploy:update_code", "rubber:passenger:remove_config_ru" if Rubber::Util.is_rails?
+
+ task :remove_config_ru, :roles => :passenger do
+ rsudo "rm -f #{releases_path}/*/config.ru"
+ end
+
+ # passenger depends on apache for start/stop/restart, just need these defined
+ # as apache hooks into standard deploy lifecycle
+
+ deploy.task :restart, :roles => :app do
+ end
+
+ deploy.task :stop, :roles => :app do
+ end
+
+ deploy.task :start, :roles => :app do
+ end
+
+ end
+end
140 config/rubber/deploy-postgresql.rb
View
@@ -0,0 +1,140 @@
+
+namespace :rubber do
+
+ namespace :postgresql do
+
+ rubber.allow_optional_tasks(self)
+
+ before "rubber:install_packages", "rubber:postgresql:setup_apt_sources"
+
+ task :setup_apt_sources do
+ rubber.sudo_script 'configure_postgresql_repository', <<-ENDSCRIPT
+ # PostgreSQL 9.1 is the default starting in Ubuntu 11.10.
+ release=`lsb_release -sr`
+ needs_repo=`echo "$release < 11.10" | bc`
+ if [[ $needs_repo == 1 ]]; then
+ add-apt-repository ppa:pitti/postgresql
+ fi
+ ENDSCRIPT
+ end
+
+ after "rubber:create", "rubber:postgresql:validate_db_roles"
+
+ task :validate_db_roles do
+ db_instances = rubber_instances.for_role("postgresql_slave")
+ db_instances.each do |instance|
+ if instance.role_names.find {|n| n == 'postgresql_master'}
+ fatal "Cannot have a postgresql slave and master on the same instance, please remove slave role for #{instance.name}"
+ end
+ end
+ end
+
+ after "rubber:bootstrap", "rubber:postgresql:bootstrap"
+
+ # Bootstrap the production database config. Db bootstrap is special - the
+ # user could be requiring the rails env inside some of their config
+ # templates, which creates a catch 22 situation with the db, so we try and
+ # bootstrap the db separate from the rest of the config
+ task :bootstrap, :roles => [:postgresql_master, :postgresql_slave] do
+
+ # Conditionally bootstrap for each node/role only if that node has not
+ # been bootstrapped for that role before
+ master_instances = rubber_instances.for_role("postgresql_master") & rubber_instances.filtered
+ master_instances.each do |ic|
+ task_name = "_bootstrap_postgresql_master_#{ic.full_name}".to_sym()
+ task task_name, :hosts => ic.full_name do
+ env = rubber_cfg.environment.bind("postgresql_master", ic.name)
+ exists = capture("echo $(ls #{env.postgresql_data_dir}/ 2> /dev/null)")
+ if exists.strip.size == 0
+ common_bootstrap
+ sudo "/usr/lib/postgresql/#{rubber_env.postgresql_ver}/bin/initdb -D #{rubber_env.postgresql_data_dir}", :as => 'postgres'
+ sudo "#{rubber_env.postgresql_ctl} start"
+ sleep 5
+
+ create_user_cmd = "CREATE USER #{env.db_user} WITH NOSUPERUSER CREATEDB NOCREATEROLE"
+ create_user_cmd << " PASSWORD '#{env.db_pass}'" if env.db_pass
+
+ create_replication_user_cmd = "CREATE USER #{env.db_replication_user} WITH NOSUPERUSER NOCREATEROLE REPLICATION"
+ create_replication_user_cmd << " PASSWORD '#{env.db_replication_pass}'" if env.db_replication_pass
+
+ rubber.sudo_script "create_master_db", <<-ENDSCRIPT
+ sudo -u postgres psql -c "#{create_user_cmd}"
+ sudo -u postgres psql -c "#{create_replication_user_cmd}"
+ sudo -u postgres psql -c "CREATE DATABASE #{env.db_name} WITH OWNER #{env.db_user}"
+ ENDSCRIPT
+ end
+ end
+ send task_name
+ end
+
+ slave_instances = rubber_instances.for_role("postgresql_slave") & rubber_instances.filtered
+ slave_instances.each do |ic|
+ task_name = "_bootstrap_postgresql_slave_#{ic.full_name}".to_sym()
+ task task_name, :hosts => ic.full_name do
+ env = rubber_cfg.environment.bind("postgresql_slave", ic.name)
+ exists = capture("echo $(ls #{env.postgresql_data_dir}/ 2> /dev/null)")
+ if exists.strip.size == 0
+ common_bootstrap
+ master = rubber_instances.for_role("postgresql_master").first
+
+ rsudo "/usr/lib/postgresql/#{env.postgresql_ver}/bin/pg_basebackup -D #{env.postgresql_data_dir} -U #{env.db_replication_user} -h #{master.full_name}", :as => 'postgres'
+
+ # Gen just the slave-specific conf.
+ rubber.run_config(:file => "role/postgresql_slave/", :force => true, :deploy_path => release_path)
+
+ # Start up the server.
+ rsudo "#{rubber_env.postgresql_ctl} start"
+ sleep 5
+ end
+ end
+
+ send task_name
+ end
+ end
+
+ # TODO: Make the setup/update happen just once per host
+ def common_bootstrap
+ # postgresql package install starts postgresql, so stop it
+ rsudo "#{rubber_env.postgresql_ctl} stop" rescue nil
+
+ # After everything installed on machines, we need the source tree
+ # on hosts in order to run rubber:config for bootstrapping the db
+ rubber.update_code_for_bootstrap
+
+ # Gen just the conf for the given postgresql role
+ rubber.run_config(:file => "role/(db|postgresql)/", :force => true, :deploy_path => release_path)
+
+ # reconfigure postgresql so that it sets up data dir in /mnt with correct files
+ dirs = [rubber_env.postgresql_data_dir, rubber_env.postgresql_archive_dir]
+ sudo_script 'reconfigure-postgresql', <<-ENDSCRIPT
+ mkdir -p #{dirs.join(' ')}
+ chown -R postgres:postgres #{dirs.join(' ')}
+ chmod 700 #{rubber_env.postgresql_data_dir}
+ ENDSCRIPT
+ end
+
+ desc <<-DESC
+ Starts the postgresql daemons
+ DESC
+ task :start, :roles => [:postgresql_master, :postgresql_slave] do
+ rsudo "#{rubber_env.postgresql_ctl} start"
+ end
+
+ desc <<-DESC
+ Stops the postgresql daemons
+ DESC
+ task :stop, :roles => [:postgresql_master, :postgresql_slave] do
+ rsudo "#{rubber_env.postgresql_ctl} stop || true"
+ end
+
+ desc <<-DESC
+ Restarts the postgresql daemons
+ DESC
+ task :restart, :roles => [:postgresql_master, :postgresql_slave] do
+ stop
+ start
+ end
+
+ end
+
+end
107 config/rubber/deploy-setup.rb
View
@@ -0,0 +1,107 @@
+namespace :rubber do
+ namespace :base do
+
+ rubber.allow_optional_tasks(self)
+
+ before "rubber:setup_gem_sources", "rubber:base:install_rvm"
+ task :install_rvm do
+ rubber.sudo_script "install_rvm", <<-ENDSCRIPT
+ if [[ ! `rvm --version 2> /dev/null` =~ "#{rubber_env.rvm_version}" ]]; then
+ cd /tmp
+ curl -s https://raw.github.com/wayneeseguin/rvm/master/binscripts/rvm-installer -o rvm-installer
+ chmod +x rvm-installer
+ rm -f /etc/rvmrc
+ rvm_path=#{rubber_env.rvm_prefix} ./rvm-installer --version #{rubber_env.rvm_version}
+
+ # Set up the rubygems version
+ sed -i 's/rubygems_version=.*/rubygems_version=#{rubber_env.rubygems_version}/' #{rubber_env.rvm_prefix}/config/db
+
+ # Set up the rake version
+ sed -i 's/rake.*/rake -v#{rubber_env.rake_version}/' #{rubber_env.rvm_prefix}/gemsets/default.gems
+ sed -i 's/rake.*/rake -v#{rubber_env.rake_version}/' #{rubber_env.rvm_prefix}/gemsets/global.gems
+
+ # Set up the .gemrc file
+ if [[ ! -f ~/.gemrc ]]; then
+ echo "--- " >> ~/.gemrc
+ fi
+
+ if ! grep -q 'gem: ' ~/.gemrc; then
+ echo "gem: --no-ri --no-rdoc" >> ~/.gemrc
+ fi
+ fi
+ ENDSCRIPT
+ end
+
+ # ensure that the rvm profile script gets sourced by reconnecting
+ after "rubber:base:install_rvm" do
+ teardown_connections_to(sessions.keys)
+ end
+
+ after "rubber:base:install_rvm", "rubber:base:install_rvm_ruby"
+ task :install_rvm_ruby do
+ opts = get_host_options('rvm_ruby')
+
+ # sudo_script only takes a single hash with host -> VAR, so combine our
+ # two vars so we can extract them out in the bash script
+ install_opts = get_host_options('rvm_install_options')
+ install_opts.each do |k, v|
+ opts[k] = "#{opts[k]} #{v}"
+ end
+
+ install_rvm_ruby_script = <<-ENDSCRIPT
+ rvm_ver=$1
+ shift
+ install_opts=$*
+
+ if [[ ! `rvm list default 2> /dev/null` =~ "$rvm_ver" ]]; then
+ echo "RVM is compiling/installing ruby $rvm_ver, this may take a while"
+
+ nohup rvm install $rvm_ver $install_opts &> /tmp/install_rvm_ruby.log &
+ sleep 1
+
+ while true; do
+ if ! ps ax | grep -q "[r]vm install"; then break; fi
+ echo -n .
+ sleep 5
+ done
+
+ # need to set default after using once or something in env is broken
+ rvm use $rvm_ver &> /dev/null
+ rvm use $rvm_ver --default
+
+ # Something flaky with $PATH having an entry for "bin" which breaks
+ # munin, the below seems to fix it
+ rvm use $rvm_ver
+ rvm repair environments
+ rvm use $rvm_ver
+ fi
+ ENDSCRIPT
+ opts[:script_args] = '$CAPISTRANO:VAR$'
+ rubber.sudo_script "install_rvm_ruby", install_rvm_ruby_script, opts
+ end
+
+ after "rubber:install_packages", "rubber:base:configure_git" if scm == "git"
+ task :configure_git do
+ rubber.sudo_script 'configure_git', <<-ENDSCRIPT
+ if [[ "#{repository}" =~ "@" ]]; then
+ # Get host key for src machine to prevent ssh from failing
+ rm -f ~/.ssh/known_hosts
+ ssh -o 'StrictHostKeyChecking=no' #{repository.gsub(/:.*/, '')} &> /dev/null || true
+ fi
+ ENDSCRIPT
+ end
+
+ # We need a rails user for safer permissions used by deploy.rb
+ after "rubber:install_packages", "rubber:base:custom_install"
+ task :custom_install do
+ rubber.sudo_script 'custom_install', <<-ENDSCRIPT
+ # add the user for running app server with
+ if ! id #{rubber_env.app_user} &> /dev/null; then adduser --system --group #{rubber_env.app_user}; fi
+
+ # add ssh keys for root
+ if [[ ! -f /root/.ssh/id_dsa ]]; then ssh-keygen -q -t dsa -N '' -f /root/.ssh/id_dsa; fi
+ ENDSCRIPT
+ end
+
+ end
+end
91 config/rubber/deploy-util.rb
View
@@ -0,0 +1,91 @@
+namespace :rubber do
+ namespace :util do
+
+ rubber.allow_optional_tasks(self)
+
+ desc <<-DESC
+ Backup database using rubber util:backup_db
+ DESC
+ task :backup do
+ master_instances = rubber_instances.for_role('db', 'primary' => true)
+ slaves = rubber_instances.for_role('db', {})
+
+ # Select only one instance for backup. Favor slave database.
+ selected_db_instance = (slaves+master_instances).first
+
+ task_name = "_backup_db_#{selected_db_instance.full_name}".to_sym()
+ task task_name, :hosts => selected_db_instance.full_name do
+ rsudo "cd #{current_path} && RUBBER_ENV=#{Rubber.env} ./script/rubber util:backup_db --directory=/mnt/db_backups --dbuser=#{rubber_env.db_user} --dbpass=#{rubber_env.db_pass} --dbname=#{rubber_env.db_name} --dbhost=#{selected_db_instance.full_name}"
+ end
+ send task_name
+ end
+
+ desc <<-DESC
+ Restore database from cloud using rubber util:restore_db
+ DESC
+ task :restore_cloud do
+ filename = get_env('FILENAME', "The cloud key to restore", true)
+ master_instances = rubber_instances.for_role('db', 'primary' => true)
+ slaves = rubber_instances.for_role('db', {})
+
+ for instance in master_instances+slaves
+ task_name = "_restore_db_cloud_#{instance.full_name}".to_sym()
+ task task_name, :hosts => instance.full_name do
+ rsudo "cd #{current_path} && RUBBER_ENV=#{Rubber.env} ./script/rubber util:restore_db --filename=#{filename} --dbuser=#{rubber_env.db_user} --dbpass=#{rubber_env.db_pass} --dbname=#{rubber_env.db_name} --dbhost=#{instance.full_name}"
+ end
+ send task_name
+ end
+ end
+
+ desc <<-DESC
+ Overwrite production database with export from local production database.
+ DESC
+ task :local_to_cloud do
+ require 'yaml'
+ master_instances = rubber_instances.for_role('db', 'primary' => true)
+ slaves = rubber_instances.for_role('db', {})
+
+ # Select only one instance for backup. Favor slave database.
+ selected_db_instance = (slaves+master_instances).first
+
+ task_name = "_load_local_to_#{selected_db_instance.full_name}".to_sym()
+ task task_name, :hosts => selected_db_instance.full_name do
+
+ # Dump Local to tmp folder
+ filename = "#{application}.local.#{Time.now.to_i}.sql.gz"
+ backup_file = "/tmp/#{filename}"
+ on_rollback { delete file }
+ FileUtils.mkdir_p(File.dirname(backup_file))
+
+ # Use database.yml to get connection params
+ db = YAML::load(ERB.new(IO.read(File.join(File.dirname(__FILE__), '..','database.yml'))).result)[Rubber.env]
+ user = db['username']
+ pass = db['password']
+ pass = nil if pass and pass.strip.size == 0
+ host = db['host']
+ name = db['database']
+
+ raise "No db_backup_cmd defined in rubber.yml, cannot backup!" unless rubber_env.db_backup_cmd
+ db_backup_cmd = rubber_env.db_backup_cmd.gsub(/%([^%]+)%/, '#{\1}')
+ db_backup_cmd = eval('%Q{' + db_backup_cmd + '}')
+
+ # dbdump (or backup app) needs to be in your path
+ puts "Backing up database with command:"
+ system(db_backup_cmd)
+ puts "Created backup: #{backup_file}"
+
+ # Upload Local to Cloud
+ backup_bucket = Rubber.cloud.env.backup_bucket
+ dest = "db/#{File.basename(backup_file)}"
+
+ puts "Saving db dump to cloud: #{backup_bucket}:#{dest}"
+ Rubber.cloud.storage(backup_bucket).store(dest, open(backup_file))
+
+ send :restore_cloud
+
+ end
+ send task_name
+ end
+
+ end
+end
10 config/rubber/role/apache/deflate.conf
View
@@ -0,0 +1,10 @@
+<%
+ @path = "/etc/apache2/mods-available/deflate.conf"
+ @post = "a2enmod deflate"
+%>
+<IfModule mod_deflate.c>
+ SetOutputFilter DEFLATE
+ BrowserMatch ^Mozilla/4 gzip-only-text/html
+ BrowserMatch ^Mozilla/4.0[678] no-gzip
+ BrowserMatch bMSIE !no-gzip !gzip-only-text/html
+</IfModule>
9 config/rubber/role/apache/expires.conf
View
@@ -0,0 +1,9 @@
+<%
+ @path = "/etc/apache2/mods-available/expires.conf"
+ @post = "a2enmod expires"
+%>
+ExpiresActive On
+<FilesMatch "\.(ico|gif|jpe?g|png|js|css)$">
+ ExpiresDefault "access plus 1 year"
+</FilesMatch>
+
6 config/rubber/role/apache/headers.conf
View
@@ -0,0 +1,6 @@
+<%
+ @path = "/etc/apache2/mods-available/headers.conf"
+ @post = "a2enmod headers"
+%>
+# Make sure proxies don't deliver the wrong content
+Header append Vary User-Agent env=!dont-vary
7 config/rubber/role/apache/monit-apache.conf
View
@@ -0,0 +1,7 @@
+<%
+ @path = '/etc/monit/monit.d/monit-apache.conf'
+%>
+check process apache with pidfile /var/run/apache2.pid
+ group apache-<%= Rubber.env %>
+ start program = "/usr/bin/env service apache2 start"
+ stop program = "/usr/bin/env service apache2 stop"
8 config/rubber/role/apache/ports.conf
View
@@ -0,0 +1,8 @@
+<%
+ @path = '/etc/apache2/ports.conf'
+ # need to stop here so that apache will release port 80 before other
+ # services start and need to bind to it (e.g. haproxy)
+ @post = 'service apache2 stop'
+%>
+# empty ports file since other modules contribute which ports to open (vhosts)
+
52 config/rubber/role/apache/setenvif.conf
View
@@ -0,0 +1,52 @@
+<%
+ @path = "/etc/apache2/mods-available/setenvif.conf"
+ @post = "a2enmod setenvif"
+%>
+<IfModule mod_setenvif.c>
+
+#
+# The following directives modify normal HTTP response behavior to
+# handle known problems with browser implementations.
+#
+BrowserMatch "Mozilla/2" nokeepalive
+BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0
+BrowserMatch "RealPlayer 4\.0" force-response-1.0
+BrowserMatch "Java/1\.0" force-response-1.0
+BrowserMatch "JDK/1\.0" force-response-1.0
+
+#
+# The following directive disables redirects on non-GET requests for
+# a directory that does not include the trailing slash. This fixes a
+# problem with Microsoft WebFolders which does not appropriately handle
+# redirects for folders with DAV methods.
+# Same deal with Apple's DAV filesystem and Gnome VFS support for DAV.
+#
+BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully
+BrowserMatch "MS FrontPage" redirect-carefully
+BrowserMatch "^WebDrive" redirect-carefully
+BrowserMatch "^WebDAVFS/1.[012]" redirect-carefully
+BrowserMatch "^gnome-vfs/1.0" redirect-carefully
+BrowserMatch "^XML Spy" redirect-carefully
+BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
+
+SetEnvIf User-Agent "^websitepulse.*" no-log
+
+# Netscape 4.x has some problems...
+ BrowserMatch ^Mozilla/4 gzip-only-text/html
+
+ # Netscape 4.06-4.08 have some more problems
+ BrowserMatch ^Mozilla/4\.0[678] no-gzip
+
+ # MSIE masquerades as Netscape, but it is fine
+ BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
+
+ # Don't compress images
+ SetEnvIfNoCase Request_URI \
+ \.(?:gif|jpe?g|png)$ no-gzip dont-vary
+
+ # Don't compress downloads from files that are
+ # downloaded before they have been pushed to S3.
+ SetEnvIfNoCase Request_URI ^/assets/.* no-gzip dont-vary
+
+</IfModule>
+
21 config/rubber/role/collectd/collectd-ping.conf
View
@@ -0,0 +1,21 @@
+<%
+ @path = '/etc/collectd/conf.d/collectd-ping.conf'
+
+ ping_hosts = []
+ if rubber_instances[rubber_env.host].role_names.include?('graphite_server')
+ # monitor all servers from graphite_server
+ ping_hosts = rubber_instances.collect {|ic| ic.full_name }.select {|h| h != rubber_env.full_host }
+ elsif rubber_instances[rubber_env.host].role_names.include?('web_tools')
+ # monitor just graphite_server from tools
+ ping_hosts = Array((rubber_instances.for_role('graphite_server').first.full_name rescue nil))
+ end
+
+ @skip = (ping_hosts.size == 0)
+%>
+
+LoadPlugin ping
+<Plugin "ping">
+ <% ping_hosts.each do |h| %>
+ Host "<%= h %>"
+ <% end %>
+</Plugin>
69 config/rubber/role/collectd/collectd.conf
View
@@ -0,0 +1,69 @@
+<%
+ rubber_scripts = `#{Rubber.root}/#{rubber_env.rubber_collectd_runner} config`.split("\n") rescue []
+
+ @path = '/etc/collectd/collectd.conf'
+ @post = "mkdir -p /etc/collectd/conf.d/ /etc/collectd/thresholds.d/ /etc/collectd/filters.d/ /etc/collectd/plugins/ "
+%>
+# Config file for collectd(1). Generated by Rubber
+#
+# Some plugins need additional configuration and are disabled by default.
+# Please read collectd.conf(5) for details.
+#
+# You should also read /usr/share/doc/collectd-core/README.Debian.plugins
+# before enabling any more plugins.
+
+Hostname "<%= rubber_env.host %>"
+#BaseDir "/var/lib/collectd"
+#PluginDir "/usr/lib/collectd"
+TypesDB "/usr/share/collectd/types.db" "/etc/collectd/types.db"
+Interval 60
+#Timeout 2
+#ReadThreads 5
+
+LoadPlugin syslog
+
+<Plugin syslog>
+ LogLevel info
+</Plugin>
+
+LoadPlugin cpu
+LoadPlugin df
+LoadPlugin disk
+LoadPlugin exec
+LoadPlugin entropy
+LoadPlugin interface
+LoadPlugin irq
+LoadPlugin load
+LoadPlugin memory
+LoadPlugin processes
+#LoadPlugin python
+LoadPlugin rrdtool
+<% if `swapon -s | wc -l`.to_i > 1 %>
+LoadPlugin swap
+<% end %>
+#LoadPlugin table
+#LoadPlugin tail
+#LoadPlugin tcpconns
+LoadPlugin users
+
+LoadPlugin notify_email
+
+<Plugin "notify_email">
+ From "collectd@<%= rubber_env.full_host %>"
+ Recipient "<%= rubber_env.admin_email %>"
+ Subject "Collectd notify: %s@%s"
+</Plugin>
+
+<%
+ if rubber_scripts.size > 0
+%>
+
+<Plugin exec>
+ Exec <%= rubber_env.app_user %> "/bin/bash" "-l" "-c" "cd <%= "/mnt/#{rubber_env.app_name}-#{Rubber.env}/current" %> && bundle exec <%= rubber_env.rubber_collectd_runner %>"
+</Plugin>
+
+<% end %>
+
+Include "/etc/collectd/conf.d/*.conf"
+Include "/etc/collectd/thresholds.d/*.conf"
+Include "/etc/collectd/filters.d/*.conf"
35 config/rubber/role/collectd/filters.conf
View
@@ -0,0 +1,35 @@
+<%
+ @path = '/etc/collectd/filters.d/filters.conf'
+%>
+# Filter configuration for collectd(1).
+#
+# See the section "FILTER CONFIGURATION" in collectd.conf(5) for details.
+
+#PreCacheChain "PreCache"
+#PostCacheChain "PostCache"
+
+#LoadPlugin match_empty_counter
+#LoadPlugin match_hashed
+#LoadPlugin match_regex
+#LoadPlugin match_timediff
+#LoadPlugin match_value
+
+#LoadPlugin target_notification
+#LoadPlugin target_replace
+#LoadPlugin target_scale
+#LoadPlugin target_set
+
+#<Chain "PreCache">
+# <Rule "no_fqdn">
+# <Match "regex">
+# Host "^[^\.]*$"
+# Invert false
+# </Match>
+# Target "stop"
+# </Rule>
+#</Chain>
+
+#<Chain "PostCache">
+# Target "write"
+#</Chain>
+
21 config/rubber/role/collectd/graphite-collectd.conf
View
@@ -0,0 +1,21 @@
+<%
+ graphite_server = rubber_instances.for_role('graphite_server').first.full_name rescue nil
+ @path = '/etc/collectd/conf.d/graphite.conf'
+ @skip = graphite_server.nil?
+%>
+
+<LoadPlugin "perl">
+ Globals true
+</LoadPlugin>
+
+<Plugin "perl">
+ BaseName "Collectd::Plugins"
+ LoadPlugin "Graphite"
+
+ <Plugin "Graphite">
+ Buffer "<%= Rubber.env == 'production' ? 1024 : 0 %>"
+ Prefix "servers"
+ Host "<%= graphite_server %>"
+ Port "<%= rubber_env.graphite_server_port %>"
+ </Plugin>
+</Plugin>
88 config/rubber/role/collectd/thresholds.conf
View
@@ -0,0 +1,88 @@
+<%
+ is_worker_instance = rubber_instances[rubber_env.host].role_names.include?('resque_worker')
+
+ @path = '/etc/collectd/thresholds.d/thresholds.conf'
+%>
+# Threshold configuration for collectd(1).
+#
+# See the section "THRESHOLD CONFIGURATION" in collectd.conf(5) for details.
+
+<Threshold>
+
+ <Type "df">
+ DataSource "used"
+ WarningMax 80
+ FailureMax 95
+ Percentage true
+ </Type>
+
+ <Type "load">
+ DataSource "midterm"
+ WarningMax <%= is_worker_instance ? 100 : 20 %>
+ Hysteresis 0.3
+ </Type>
+
+ <Plugin "ping">
+ <Type "ping">
+ Datasource "value"
+ WarningMax 50
+ FailureMax 100
+ </Type>
+ <Type "ping_droprate">
+ Datasource "value"
+ FailureMax 0.05
+ </Type>
+ </Plugin>
+
+</Threshold>
+
+#<Threshold>
+# <Type "counter">
+# WarningMin 0.00
+# WarningMax 1000.00
+# FailureMin 0
+# FailureMax 1200.00
+# Invert false
+# Persist false
+# Instance "some_instance"
+# </Type>
+#
+# <Type "df">
+# WarningMax 90
+# Percentage true
+# </Type>
+#
+# <Type "load">
+# DataSource "midterm"
+# WarningMax 1
+# Hysteresis 0.3
+# </Type>
+#
+# <Type "cpu">
+# Instance "user"
+# WarningMax 85
+# Hits 6
+# </Type>
+#
+# <Plugin "interface">
+# Instance "eth0"
+# <Type "if_octets">
+# DataSource "rx"
+# FailureMax 10000000
+# </Type>
+# </Plugin>
+#
+# <Host "hostname">
+# <Type "cpu">
+# Instance "idle"
+# FailureMin 10
+# </Type>
+#
+# <Plugin "memory">
+# <Type "memory">
+# Instance "cached"
+# WarningMin 100000000
+# </Type>
+# </Plugin>
+# </Host>
+#</Threshold>
5 config/rubber/role/collectd/types.db
View
@@ -0,0 +1,5 @@
+<%
+ @path = '/etc/collectd/types.db'
+%>
+
+domains_by_segment domain_count:GAUGE:0:U, avg_user_count:GAUGE:0:U, avg_storage:GAUGE:0:U
232 config/rubber/role/graphite_server/carbon.conf
View
@@ -0,0 +1,232 @@
+<%
+ @path = '/opt/graphite/conf/carbon.conf'
+%>
+[cache]
+# Configure carbon directories.
+#
+# OS environment variables can be used to tell carbon where graphite is
+# installed, where to read configuration from and where to write data.
+#
+# GRAPHITE_ROOT - Root directory of the graphite installation.
+# Defaults to ../
+# GRAPHITE_CONF_DIR - Configuration directory (where this file lives).
+# Defaults to $GRAPHITE_ROOT/conf/
+# GRAPHITE_STORAGE_DIR - Storage directory for whipser/rrd/log/pid files.
+# Defaults to $GRAPHITE_ROOT/storage/
+#
+# To change other directory paths, add settings to this file. The following
+# configuration variables are available with these default values:
+#
+# STORAGE_DIR = $GRAPHITE_STORAGE_DIR
+# LOCAL_DATA_DIR = STORAGE_DIR/whisper/
+# WHITELISTS_DIR = STORAGE_DIR/lists/
+# CONF_DIR = STORAGE_DIR/conf/
+# LOG_DIR = STORAGE_DIR/log/
+# PID_DIR = STORAGE_DIR/
+#
+# For FHS style directory structures, use:
+#
+# STORAGE_DIR = /var/lib/carbon/
+# CONF_DIR = /etc/carbon/
+# LOG_DIR = /var/log/carbon/
+# PID_DIR = /var/run/
+#
+LOCAL_DATA_DIR = /opt/graphite/storage/whisper/
+
+# Specify the user to drop privileges to
+# If this is blank carbon runs as the user that invokes it
+# This user must have write access to the local data directory
+USER =
+
+# Limit the size of the cache to avoid swapping or becoming CPU bound.
+# Sorts and serving cache queries gets more expensive as the cache grows.
+# Use the value "inf" (infinity) for an unlimited cache size.
+MAX_CACHE_SIZE = inf
+
+# Limits the number of whisper update_many() calls per second, which effectively
+# means the number of write requests sent to the disk. This is intended to
+# prevent over-utilizing the disk and thus starving the rest of the system.
+# When the rate of required updates exceeds this, then carbon's caching will
+# take effect and increase the overall throughput accordingly.
+MAX_UPDATES_PER_SECOND = 1000
+
+# Softly limits the number of whisper files that get created each minute.
+# Setting this value low (like at 50) is a good way to ensure your graphite
+# system will not be adversely impacted when a bunch of new metrics are
+# sent to it. The trade off is that it will take much longer for those metrics'
+# database files to all get created and thus longer until the data becomes usable.
+# Setting this value high (like "inf" for infinity) will cause graphite to create
+# the files quickly but at the risk of slowing I/O down considerably for a while.
+MAX_CREATES_PER_MINUTE = 50
+
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = <%= rubber_env.graphite_server_port %>
+
+# Set this to True to enable the UDP listener. By default this is off
+# because it is very common to run multiple carbon daemons and managing
+# another (rarely used) port for every carbon instance is not fun.
+ENABLE_UDP_LISTENER = False
+UDP_RECEIVER_INTERFACE = 0.0.0.0
+UDP_RECEIVER_PORT = 2003
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2004
+
+# Per security concerns outlined in Bug #817247 the pickle receiver
+# will use a more secure and slightly less efficient unpickler.
+# Set this to True to revert to the old-fashioned insecure unpickler.
+USE_INSECURE_UNPICKLER = False
+
+CACHE_QUERY_INTERFACE = 0.0.0.0
+CACHE_QUERY_PORT = 7002
+
+# Set this to False to drop datapoints received after the cache
+# reaches MAX_CACHE_SIZE. If this is True (the default) then sockets
+# over which metrics are received will temporarily stop accepting
+# data until the cache size falls below 95% MAX_CACHE_SIZE.
+USE_FLOW_CONTROL = True
+
+# By default, carbon-cache will log every whisper update. This can be excessive and
+# degrade performance if logging on the same volume as the whisper data is stored.
+LOG_UPDATES = False
+
+# On some systems it is desirable for whisper to write synchronously.
+# Set this option to True if you'd like to try this. Basically it will
+# shift the onus of buffering writes from the kernel into carbon's cache.
+WHISPER_AUTOFLUSH = False
+
+# Enable AMQP if you want to receve metrics using an amqp broker
+# ENABLE_AMQP = False
+
+# Verbose means a line will be logged for every metric received
+# useful for testing
+# AMQP_VERBOSE = False
+
+# AMQP_HOST = localhost
+# AMQP_PORT = 5672
+# AMQP_VHOST = /
+# AMQP_USER = guest
+# AMQP_PASSWORD = guest
+# AMQP_EXCHANGE = graphite
+# AMQP_METRIC_NAME_IN_BODY = False
+
+# The manhole interface allows you to SSH into the carbon daemon
+# and get a python interpreter. BE CAREFUL WITH THIS! If you do
+# something like time.sleep() in the interpreter, the whole process
+# will sleep! This is *extremely* helpful in debugging, assuming
+# you are familiar with the code. If you are not, please don't
+# mess with this, you are asking for trouble :)
+#
+# ENABLE_MANHOLE = False
+# MANHOLE_INTERFACE = 127.0.0.1
+# MANHOLE_PORT = 7222
+# MANHOLE_USER = admin
+# MANHOLE_PUBLIC_KEY = ssh-rsa AAAAB3NzaC1yc2EAAAABiwAaAIEAoxN0sv/e4eZCPpi3N3KYvyzRaBaMeS2RsOQ/cDuKv11dlNzVeiyc3RFmCv5Rjwn/lQ79y0zyHxw67qLyhQ/kDzINc4cY41ivuQXm2tPmgvexdrBv5nsfEpjs3gLZfJnyvlcVyWK/lId8WUvEWSWHTzsbtmXAF2raJMdgLTbQ8wE=
+
+# Patterns for all of the metrics this machine will store. Read more at
+# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
+#
+# Example: store all sales, linux servers, and utilization metrics
+# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
+#
+# Example: store everything
+# BIND_PATTERNS = #
+
+# To configure special settings for the 'foo' carbon-cache instance, uncomment this:
+#[cache:foo]
+#LINE_RECEIVER_PORT = 2103
+#PICKLE_RECEIVER_PORT = 2104
+#CACHE_QUERY_PORT = 7102
+# and any other settings you want to customize, defaults are inherited
+# from [carbon] section.
+
+
+[relay]
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2013
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2014
+
+# To use consistent hashing instead of the user defined relay-rules.conf,
+# change this to:
+# RELAY_METHOD = consistent-hashing
+RELAY_METHOD = rules
+
+# If you use consistent-hashing you may want to add redundancy
+# of your data by replicating every datapoint to more than
+# one machine.
+REPLICATION_FACTOR = 1
+
+# This is a list of carbon daemons we will send any relayed or
+# generated metrics to. The default provided would send to a single
+# carbon-cache instance on the default port. However if you
+# use multiple carbon-cache instances then it would look like this:
+#
+# DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b
+#
+# The general form is IP:PORT:INSTANCE where the :INSTANCE part is
+# optional and refers to the "None" instance if omitted.
+#
+# Note that if the destinations are all carbon-caches then this should
+# exactly match the webapp's CARBONLINK_HOSTS setting in terms of
+# instances listed (order matters!).
+DESTINATIONS = 127.0.0.1:2004
+
+# This defines the maximum "message size" between carbon daemons.
+# You shouldn't need to tune this unless you really know what you're doing.
+MAX_DATAPOINTS_PER_MESSAGE = 500
+MAX_QUEUE_SIZE = 10000
+
+# Set this to False to drop datapoints when any send queue (sending datapoints
+# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
+# default) then sockets over which metrics are received will temporarily stop accepting
+# data until the send queues fall below 80% MAX_QUEUE_SIZE.
+USE_FLOW_CONTROL = True
+
+
+[aggregator]
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2023
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2024
+
+# This is a list of carbon daemons we will send any relayed or
+# generated metrics to. The default provided would send to a single
+# carbon-cache instance on the default port. However if you
+# use multiple carbon-cache instances then it would look like this:
+#
+# DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b
+#
+# The format is comma-delimited IP:PORT:INSTANCE where the :INSTANCE part is
+# optional and refers to the "None" instance if omitted.
+#
+# Note that if the destinations are all carbon-caches then this should
+# exactly match the webapp's CARBONLINK_HOSTS setting in terms of
+# instances listed (order matters!).
+DESTINATIONS = 127.0.0.1:2004
+
+# If you want to add redundancy to your data by replicating every
+# datapoint to more than one machine, increase this.
+REPLICATION_FACTOR = 1
+
+# This is the maximum number of datapoints that can be queued up
+# for a single destination. Once this limit is hit, we will
+# stop accepting new data if USE_FLOW_CONTROL is True, otherwise
+# we will drop any subsequently received datapoints.
+MAX_QUEUE_SIZE = 10000
+
+# Set this to False to drop datapoints when any send queue (sending datapoints
+# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
+# default) then sockets over which metrics are received will temporarily stop accepting
+# data until the send queues fall below 80% MAX_QUEUE_SIZE.
+USE_FLOW_CONTROL = True
+
+# This defines the maximum "message size" between carbon daemons.
+# You shouldn't need to tune this unless you really know what you're doing.
+MAX_DATAPOINTS_PER_MESSAGE = 500
+
+# This defines how many datapoints the aggregator remembers for
+# each metric. Aggregation only happens for datapoints that fall in
+# the past MAX_AGGREGATION_INTERVALS * intervalSize seconds.
+MAX_AGGREGATION_INTERVALS = 5
16 config/rubber/role/graphite_server/graphite_server-upstart.conf
View
@@ -0,0 +1,16 @@
+<%
+ @path = "/etc/init/graphite-server.conf"
+ @backup = false
+%>
+description "graphite server"
+
+start on [2345]
+stop on runlevel [016]
+
+expect daemon
+
+script
+ cd <%= rubber_env.graphite_dir %>
+ rm -f <%= rubber_env.graphite_server_pid_file %>
+ exec ./bin/carbon-cache.py --pidfile <%= rubber_env.graphite_server_pid_file %> start
+end script
19 config/rubber/role/graphite_server/storage-schemas.conf
View
@@ -0,0 +1,19 @@
+<%
+ @path = '/opt/graphite/conf/storage-schemas.conf'
+%>
+
+# Schema definitions for whisper files. Entries are scanned in order,
+# and first match wins.
+#
+# [name]
+# pattern = regex
+# retentions = timePerPoint:timeToStore, timePerPoint:timeToStore, ...
+#
+# [default_1min_for_1day]
+# pattern = .*
+# retentions = 60s:1d
+
+[everything_1min_13months]
+priority = 100
+pattern = .*
+retentions = 60:565920
8 config/rubber/role/graphite_web/crontab
View
@@ -0,0 +1,8 @@
+<%
+ @read_cmd = 'crontab -l'
+ @write_cmd = 'crontab -'
+ @additive = ['# start-graphite_web-crontab', '# end-graphite_web-crontab']
+%>
+
+# Backup graphite web UI db every day
+0 1 * * * <%= Rubber.root %>/script/rubber cron --task util:backup --directory /mnt/graphite_backups --name graphite --command "nice zip \%dir\%/\%name\%_<%= rubber_env.host %>_\%time_stamp\%.zip /opt/graphite/storage/*.db"
60 config/rubber/role/graphite_web/dashboard.conf
View
@@ -0,0 +1,60 @@
+<%
+ @path = '/opt/graphite/conf/dashboard.conf'
+%>
+# This configuration file controls the behavior of the Dashboard UI, available
+# at http://my-graphite-server/dashboard/.
+#
+# This file must contain a [ui] section that defines values for all of the
+# following settings.
+[ui]
+default_graph_width = 400
+default_graph_height = 250
+automatic_variants = true
+refresh_interval = 60
+autocomplete_delay = 375
+merge_hover_delay = 750
+
+# You can set this 'default', 'white', or a custom theme name.
+# To create a custom theme, copy the dashboard-default.css file
+# to dashboard-myThemeName.css in the content/css directory and
+# modify it to your liking.
+theme = default
+
+[keyboard-shortcuts]
+toggle_toolbar = ctrl-z
+toggle_metrics_panel = ctrl-space
+erase_all_graphs = alt-x
+save_dashboard = alt-s
+completer_add_metrics = alt-enter
+completer_del_metrics = alt-backspace
+give_completer_focus = shift-space
+
+# These settings apply to the UI as a whole, all other sections in this file
+# pertain only to specific metric types.
+#
+# The dashboard presents only metrics that fall into specified naming schemes
+# defined in this file. This creates a simpler, more targetted view of the
+# data. The general form for defining a naming scheme is as follows:
+#
+#[Metric Type]
+#scheme = basis.path.<field1>.<field2>.<fieldN>
+#field1.label = Foo
+#field2.label = Bar
+#
+#
+# Where each <field> will be displayed as a dropdown box
+# in the UI and the remaining portion of the namespace
+# shown in the Metric Selector panel. The .label options set the labels
+# displayed for each dropdown.
+#
+# For example:
+#
+#[Sales]
+#scheme = sales.<channel>.<type>.<brand>
+#channel.label = Channel
+#type.label = Product Type
+#brand.label = Brand
+#
+# This defines a 'Sales' metric type that uses 3 dropdowns in the Context Selector
+# (the upper-left panel) while any deeper metrics (per-product counts or revenue, etc)
+# will be available in the Metric Selector (upper-right panel).
385 config/rubber/role/graphite_web/dashboard.html
View
@@ -0,0 +1,385 @@
+<!--
+<%
+ @path = "/opt/graphite/webapp/content/dashboard.html"
+%>
+
+Simple graphite dashboard cycler for use on a large display - e.g. googletv/chrome on a wall mounted TV
+
+-->
+
+<html>
+<head>
+ <title>Graphite TV Dashboards</title>
+
+ <style type="text/css">
+ body {
+ background-color: black;
+ }
+
+ .dashboard-title {
+ color: yellow;
+ text-align: center;
+ }
+
+ #paused_popup {
+ display: none;
+ background-color: gray;
+ color: black;
+ position: absolute;
+ top: 0%;
+ left: 0%;
+ padding: 10px;
+ }
+ </style>
+
+ <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.js"></script>
+
+ <script type="text/javascript">
+
+ // To test this locally, edit ROOT_URL to point to graphite webapp (without trailing slash)
+ // To allow xhttp to graphite from html loaded from local file, run chrome with:
+ // /Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --disable-web-security
+
+ var ROOT_URL = window.location.href.indexOf('file:') >= 0 ? "https://:8443" : "";
+
+ function Dashboards(reloadAfterCycles, pageCount) {
+ var self = this;
+
+ self.reloadAfterCycles = reloadAfterCycles;
+ self.pageCount = pageCount;
+ self.cycleCount = 0;
+ self.pages = [];
+ self.pagesTmp = [];
+ self.pageIdx = -1;
+
+ self.reloadDashboards = function() {
+ self.pages = [];
+ self.pagesTmp = [];
+ return $.when(self.dashboardList()).pipe(self.detailFromDashboardList).then(function() {
+ console.log("Committing dashboard pages");
+
+ self.pagesTmp.sort(function(x,y) {
+ if (x.name() < y.name())
+ return -1;
+ if (x.name() > y.name())
+ return 1;
+ return 0;
+ });
+
+ $.each(self.pagesTmp, function(i, dashboard) {
+ if (dashboard.graphCount() > self.pageCount)
+ {
+ var totalPages = Math.ceil(dashboard.graphCount() / self.pageCount);
+ console.log("Splitting dashboard " + dashboard.name() + " into " + totalPages + " pages");
+
+ for(i=0; i < totalPages; i++)
+ {
+ var name = dashboard.name() + " " + (i+1) + "/" + totalPages;
+ self.pages.push(dashboard.subset(name, i * self.pageCount, self.pageCount));
+ console.log("Committed " + name + " " + i * self.pageCount + " " + self.pageCount);
+ }
+ }
+ else
+ {
+ self.pages.push(dashboard);
+ }
+ });
+
+ });
+ }
+
+ self.dashboardList = function() {
+ console.log("Fetching all dashboards");
+
+ return $.getJSON(ROOT_URL + '/dashboard/find/', {'query': ''});
+ }
+
+ self.detailFromDashboardList = function(data) {
+ console.log("Got all dashboards");
+ console.log(data);
+
+ var chain = [];
+ var dashboards = data["dashboards"];
+ $.each(dashboards, function(i, dashboard) {
+ var detailRequest = self.dashboardDetail(dashboard);
+ chain.push(detailRequest);
+ });
+ return $.when.apply($, chain);
+ }
+
+ self.dashboardDetail = function(dashboard) {
+ var name = dashboard["name"];
+ console.log("Fetching dashboard detail for " + name);
+
+ return $.getJSON(ROOT_URL + '/dashboard/load/' + name, {}, function(data) {
+
+ console.log("Got dashboard detail for " + name);
+ console.log(data);
+
+ self.pagesTmp.push(new Dashboard(data));
+ });
+ }
+
+ self.renderNextPage = function() {
+ if (self.pages.length == 0)
+ {
+ console.log("No dashboard pages to render");
+ return;
+ }
+
+ // always wrap around page idx when it gets too big
+ self.pageIdx = (self.pageIdx + 1) % self.pages.length;
+
+ console.log("Rendering next page " + self.pageIdx + "/" + self.pages.length);
+ var result = self.pages[self.pageIdx].render();
+
+ // reload dashboards on last page so its ready when we wrap around
+ if (self.pageIdx == (self.pages.length - 1))
+ {
+ self.cycleCount++;
+ if ((self.cycleCount % self.reloadAfterCycles) == 0)
+ {
+ self.reloadDashboards();
+ }
+ }
+
+ return result;
+ }
+
+ self.renderPrevPage = function() {
+ if (self.pages.length == 0)
+ {
+ console.log("No dashboard pages to render");
+ return;
+ }
+
+ // always wrap around page idx when it gets too big
+ self.pageIdx = self.pageIdx - 1
+ if (self.pageIdx < 0) self.pageIdx = self.pages.length - 1;
+
+ console.log("Rendering prev page " + self.pageIdx + "/" + self.pages.length);
+ var result = self.pages[self.pageIdx].render();
+
+ // reload dashboards on last page so its ready when we wrap around
+ if (self.pageIdx == 0)
+ {
+ self.cycleCount++;
+ if ((self.cycleCount % self.reloadAfterCycles) == 0)
+ {
+ self.reloadDashboards();
+ }
+ return;
+ }
+
+ return result;
+ }
+ }
+
+ function Dashboard(apiData) {
+ var self = this;
+
+ self.apiData = apiData;
+
+ self.name = function() {
+ return self.apiData["state"]["name"];
+ }
+
+ self.graphCount = function() {
+ return self.apiData["state"]["graphs"].length;
+ }
+
+ self.subset = function(newName, startIdx, howMany) {
+ var newData = $.extend(true, {}, self.apiData);
+ newData["state"]["name"] = newName;
+ newData["state"]["graphs"] = newData["state"]["graphs"].splice(startIdx, howMany);
+ var newDash = new Dashboard(newData);
+ return newDash;
+ }
+
+ self.extractUrls = function() {
+ var defaultParams = self.apiData["state"]["defaultGraphParams"];
+ defaultParams = $.extend(true, {}, defaultParams); // deep copy
+
+ defaultParams["width"] = "1050";
+ defaultParams["height"] = "550";
+ defaultParams["lineWidth"] = "3";
+
+ if (! defaultParams["from"]) defaultParams["from"] = "-24hours";
+ if (! defaultParams["until"]) defaultParams["until"] = "now";
+ if (! defaultParams["title"]) defaultParams["title"] = "No Title";
+
+ var urls = $.map(self.apiData["state"]["graphs"], function(e, i) {
+ var defn = e[1];
+ var params = $.extend(true, {}, defaultParams); // deep copy
+ params = $.extend(true, params, defn); // deep copy
+ params["target"] = params["target"][0];
+
+ var url = ROOT_URL + "/render?" + $.param(params);
+
+ return url;
+ });
+ return urls;
+ }
+
+ self.render = function()
+ {
+ var name = self.apiData["state"]["name"];
+ var dbdiv = $('<div class="dashboard"/>');
+
+ var title = $('<div class="dashboard-title"/>')
+ dbdiv.append(title);
+ title.append(name);
+
+ var images = $('<div class="dashboard-images"/>');
+ dbdiv.append(images);
+
+ var urls = self.extractUrls();
+ $.each(urls, function(i, url) {
+ var img = $("<img/>", {'src': url});
+ images.append(img);
+ });
+
+ return dbdiv;
+ }
+
+ }
+
+ function View(dashboards, pageDelay, pauseDelay) {
+ var self = this;
+
+ self.pageDelay = pageDelay;
+ self.pauseDelay = pauseDelay;
+
+ self.eventLoopDelay = 1000;
+ self.eventLoopCount = 0;
+ self.paused = false;
+
+ self.dashboards = dashboards;
+
+ self.eventLoop = function()
+ {
+ if (self.paused)
+ {
+ console.log("paused");
+ }
+ else if(self.dashboards.pages.length == 0)
+ {
+ console.log("Waiting for dashboards");
+ self.eventLoopCount = 0;
+ }
+ else
+ {
+ var offset = (self.eventLoopCount * self.eventLoopDelay) % self.pageDelay;
+ if (offset == 0)
+ {
+ console.log("Rendering next page")
+ self.renderPage('next');
+ }
+ self.eventLoopCount++;
+ }
+
+ setTimeout(self.eventLoop, self.eventLoopDelay);
+ }
+
+ self.registerKeyBindings = function() {
+ $(document).keydown(function(e){
+ if (e.keyCode == 32) {
+ self.togglePause();
+ return false;
+ }
+ if (e.keyCode == 37) {
+ console.log( "left pressed" );
+ self.renderPage('prev');
+ return false;
+ }
+ if (e.keyCode == 39) {
+ console.log( "right pressed" );
+ self.renderPage('next');
+ return false;
+ }
+ });
+ }
+
+ self.togglePause = function() {
+ self.paused ? self.unPause() : self.pause();
+ if (self.paused) setTimeout(self.unPause, self.pauseDelay);
+ }
+
+ self.pause = function() {
+ self.paused = true;
+ $('#paused_popup').show();
+ console.log("paused: " + self.paused);
+ }
+
+ self.unPause = function() {
+ self.paused = false;
+ $('#paused_popup').hide();
+ console.log("paused: " + self.paused);
+ }
+
+ self.renderPage = function(direction)
+ {
+ var dbdiv = null;
+ if (direction == 'next')
+ {
+ dbdiv = self.dashboards.renderNextPage();
+ }
+ else if (direction = 'prev')
+ {
+ dbdiv = self.dashboards.renderPrevPage();
+ }
+ else
+ {
+ console.log("Bad direction: " + direction);
+ return;
+ }
+
+ if (dbdiv)
+ {
+ $('#main').fadeOut('slow', function() {
+ $(".dashboard").remove();
+ $("#main").append(dbdiv);
+ $("#main").fadeIn('slow');
+ });
+ }
+ }
+
+ }
+
+
+ function urlParams()
+ {
+ var vars = [], hash;
+ var hashes = window.location.href.slice(window.location.href.indexOf('?') + 1).split('&');
+ for(var i = 0; i < hashes.length; i++)
+ {
+ hash = hashes[i].split('=');
+ vars.push(hash[0]);
+ vars[hash[0]] = hash[1];
+ }
+ return vars;
+ }
+
+ $(document).ready(function () {
+ var reloadAfterCycles = urlParams()["reloadAfterCycles"] ? parseInt(urlParams()["reloadAfterCycles"]) : 5;
+ var pageCount = urlParams()["pageCount"] ? parseInt(urlParams()["pageCount"]) : 4;
+ var pageDelay = urlParams()["pageDelay"] ? parseInt(urlParams()["pageDelay"]) : 30000;
+ var pauseDelay = urlParams()["pauseDelay"] ? parseInt(urlParams()["pauseDelay"]) : 30000;
+
+ var dashboards = new Dashboards(reloadAfterCycles, pageCount);
+ dashboards.reloadDashboards();
+
+ var view = new View(dashboards, pageDelay, pauseDelay);
+ view.registerKeyBindings();
+ view.eventLoop();
+ });
+
+
+ </script>
+</head>
+
+<div id="main">
+ <div id="paused_popup">Paused</div>
+</div>
+
+</html>
62 config/rubber/role/graphite_web/graphite-vhost.conf
View
@@ -0,0 +1,62 @@
+<%
+ @path = '/etc/apache2/sites-available/graphite'
+ @post = <<-SCRIPT
+ a2ensite graphite
+ SCRIPT
+%>
+
+# This needs to be in your server's config somewhere, probably
+# the main httpd.conf
+# NameVirtualHost *:80
+
+# This line also needs to be in your server's config.
+# LoadModule wsgi_module modules/mod_wsgi.so
+
+# You may need to manually edit this file to fit your needs.
+# This configuration assumes the default installation prefix
+# of /opt/graphite/, if you installed graphite somewhere else
+# you will need to change all the occurances of /opt/graphite/
+# in this file to your chosen install location.
+
+Listen <%= rubber_env.graphite_web_port %>
+<VirtualHost *:<%= rubber_env.graphite_web_port %>>
+ ServerName <%= rubber_env.full_host %>
+ DocumentRoot "/opt/graphite/webapp"
+ ErrorLog /opt/graphite/storage/log/webapp/error.log
+ CustomLog /opt/graphite/storage/log/webapp/access.log common
+
+ # I've found that an equal number of processes & threads tends
+ # to show the best performance for Graphite (ymmv).
+ WSGIDaemonProcess graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120
+ WSGIProcessGroup graphite
+
+ # XXX You need to set this up!
+ # Read http://code.google.com/p/modwsgi/wiki/ConfigurationDirectives#WSGISocketPrefix
+ # WSGISocketPrefix /var/run/apache2/
+
+ # XXX You will need to create this file! There is a graphite.wsgi.example
+ # file in this directory that you can safely use, just copy it to graphite.wgsi
+ WSGIScriptAlias / /opt/graphite/conf/graphite.wsgi
+
+ Alias /content/ /opt/graphite/webapp/content/
+ <Location "/content/">
+ SetHandler None
+ </Location>
+
+ # XXX In order for the django admin site media to work you
+ # must change @DJANGO_ROOT@ to be the path to your django
+ # installation, which is probably something like:
+ # /usr/lib/python2.6/site-packages/django
+ Alias /media/ "/usr/lib/pymodules/python2.6/django/contrib/admin/media/"
+ <Location "/media/">
+ SetHandler None
+ </Location>
+
+ # The graphite.wsgi file has to be accessible by apache. It won't
+ # be visible to clients because of the DocumentRoot though.
+ <Directory /opt/graphite/conf/>
+ Order deny,allow
+ Allow from all
+ </Directory>
+
+</VirtualHost>
10 config/rubber/role/graphite_web/graphite.wsgi
View
@@ -0,0 +1,10 @@
+<%
+ @path = '/opt/graphite/conf/graphite.wsgi'
+%>
+import os, sys
+sys.path.append('/opt/graphite/webapp')
+os.environ['DJANGO_SETTINGS_MODULE'] = 'graphite.settings'
+
+import django.core.handlers.wsgi
+
+application = django.core.handlers.wsgi.WSGIHandler()
98 config/rubber/role/graphite_web/local_settings.py
View
@@ -0,0 +1,98 @@
+<%
+ @path = '/opt/graphite/webapp/graphite/local_settings.py'
+%>
+# Edit this file to override the default graphite settings, do not edit settings.py!!!
+
+# Turn on debugging and restart apache if you ever see an "Internal Server Error" page
+#DEBUG = True
+
+# Set your local timezone (django will *try* to figure this out automatically)
+# If your graphs appear to be offset by a couple hours then this probably
+# needs to be explicitly set to your local timezone.
+TIME_ZONE = 'America/New_York'
+
+# Uncomment these to enable more performance-related logging
+#LOG_RENDERING_PERFORMANCE = True
+#LOG_CACHE_PERFORMANCE = True
+
+# Override this if you need to provide documentation specific to your graphite deployment
+#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite"
+
+# Enable email-related features
+#SMTP_SERVER = "mail.mycompany.com"
+
+
+#####################################
+# LDAP Authentication Configuration #
+#####################################
+# LDAP / ActiveDirectory authentication setup
+#USE_LDAP_AUTH = True
+#LDAP_SERVER = "ldap.mycompany.com"
+#LDAP_PORT = 389
+# OR
+#LDAP_URI = "ldaps://ldap.mycompany.com:636"
+#LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com"
+#LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com"
+#LDAP_BASE_PASS = "readonly_account_password"
+#LDAP_USER_QUERY = "(username=%s)" #For Active Directory use "(sAMAccountName=%s)"
+#
+# If you want to further customize the ldap connection options you should
+# directly use ldap.set_option to set the ldap module's global options.
+# For example:
+#
+#import ldap
+#ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
+#ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, "/etc/ssl/ca")
+#ldap.set_option(ldap.OPT_X_TLS_CERTFILE, "/etc/ssl/mycert.pem")
+#ldap.set_option(ldap.OPT_X_TLS_KEYFILE, "/etc/ssl/mykey.pem")
+# See http://www.python-ldap.org/ for further details on these options.
+
+
+##########################
+# Database Configuration #
+##########################
+# By default sqlite is used. If you cluster multiple webapps you will need
+# to setup an external database (like mysql) and configure all the webapps
+# to use the same database. Note that this database is only used to store
+# django models like saved graphs, dashboards, user preferences, etc. Metric
+# data is not stored here.
+#
+# DON'T FORGET TO RUN 'manage.py syncdb' AFTER SETTING UP A NEW DB!
+#
+#DATABASE_ENGINE = 'mysql' # or 'postgres'
+#DATABASE_NAME = 'graphite'
+#DATABASE_USER = 'graphite'
+#DATABASE_PASSWORD = 'graphite-is-awesome'
+#DATABASE_HOST = 'mysql.mycompany.com'
+#DATABASE_PORT = '3306'
+
+
+#########################
+# Cluster Configuration #
+#########################
+# (To avoid excessive DNS lookups you want to stick to using IP addresses only in this entire section)
+#
+# This should list the IP address (and optionally port) of each webapp in your cluster.
+# Strings are of the form "ip[:port]"
+# Usually this will be the same as MEMCACHE_HOSTS except for the port numbers.
+#
+#CLUSTER_SERVERS = []
+
+# This lists all the memcached servers that will be used by this webapp.
+# If you have a cluster of webapps you want to make sure all of them
+# have the *exact* same value for this setting. That will maximize cache
+# efficiency. Setting MEMCACHE_HOSTS to be empty will turn off use of
+# memcached entirely.
+#
+# You should not use the loopback address 127.0.0.1 here because every webapp in
+# the cluster should use the exact same value and should list every member in the
+# cluster.
+#MEMCACHE_HOSTS = ['10.10.10.10:11211', '10.10.10.11:11211', '10.10.10.12:11211']
+
+# If you are running multiple carbon-caches on this machine (typically behind a relay using
+# consistent hashing), you'll need to list the ip address, cache query port, and instance name of each carbon-cache
+# instance on the local machine (NOT every carbon-cache in the entire cluster). The default cache query port is 7002
+# and a common scheme is to use 7102 for instance b, 7202 for instance c, etc.
+#
+# You *should* use 127.0.0.1 here.
+#CARBONLINK_HOSTS = ["127.0.0.1:7002:a", "127.0.0.1:7102:b", "127.0.0.1:7202:c"]
18 config/rubber/role/monit/monit-default.conf
View
@@ -0,0 +1,18 @@
+<%
+ @path = '/etc/default/monit'
+%>
+
+# Defaults for monit initscript
+# sourced by /etc/init.d/monit
+# installed at /etc/default/monit by maintainer scripts
+# Fredrik Steen <stone@debian.org>
+
+# You must set this variable for monit to start (Ubuntu < 11.10)
+startup=1
+
+# You must set this variable for monit to start (Ubuntu >= 11.10)
+START=yes
+
+# To change the intervals which monit should run uncomment
+# and change this variable.
+# CHECK_INTERVALS=180
8 config/rubber/role/monit/monit-postfix.conf
View
@@ -0,0 +1,8 @@
+<%
+ @path = '/etc/monit/monit.d/monit-postfix.conf'
+%>
+
+check process postfix with pidfile /var/spool/postfix/pid/master.pid
+ start program "/usr/bin/env service postfix start"
+ stop program "/usr/bin/env service postfix stop"
+ if 5 restarts within 5 cycles then timeout
252 config/rubber/role/monit/monit.conf
View
@@ -0,0 +1,252 @@
+<%
+ @path = '/etc/monit/monitrc'
+%>
+
+###############################################################################
+## Monit control file
+###############################################################################
+##
+## Comments begin with a '#' and extend through the end of the line. Keywords
+## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'.
+##
+## Bellow is the example of some frequently used statements. For information
+## about the control file, a complete list of statements and options please
+## have a look in the monit manual.
+##
+##
+###############################################################################
+## Global section
+###############################################################################
+##
+## Start monit in background (run as daemon) and check the services at 2-minute
+## intervals.
+#
+# set daemon 120
+set daemon 60
+
+#
+#
+## Set syslog logging with the 'daemon' facility. If the FACILITY option is
+## omited, monit will use 'user' facility by default. You can specify the
+## path to the file for monit native logging.
+#
+# set logfile syslog facility log_daemon
+set logfile /var/log/monit.log
+#
+#
+## Set list of mailservers for alert delivery. Multiple servers may be
+## specified using comma separator. By default monit uses port 25 - it is
+## possible to override it with the PORT option.
+#
+# set mailserver mail.bar.baz, # primary mailserver
+# backup.bar.baz port 10025, # backup mailserver on port 10025
+# localhost # fallback relay
+#
+set mailserver localhost
+
+#
+## By default monit will drop the event alert, in the case that there is no
+## mailserver available. In the case that you want to keep the events for
+## later delivery retry, you can use the EVENTQUEUE statement. The base
+## directory where undelivered events will be stored is specified by the
+## BASEDIR option. You can limit the maximal queue size using the SLOTS
+## option (if omited then the queue is limited just by the backend filesystem).
+#
+# set eventqueue
+# basedir /var/monit # set the base directory where events will be stored
+# slots 100 # optionaly limit the queue size
+#
+#
+## Monit by default uses the following alert mail format:
+##
+## --8<--
+## From: monit@$HOST # sender
+## Subject: monit alert -- $EVENT $SERVICE # subject
+##
+## $EVENT Service $SERVICE #
+## #
+## Date: $DATE #
+## Action: $ACTION #
+## Host: $HOST # body
+## Description: $DESCRIPTION #
+## #
+## Your faithful employee, #
+## monit #
+## --8<--
+##
+## You can override the alert message format or its parts such as subject
+## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc.
+## are expanded on runtime. For example to override the sender:
+#
+# set mail-format { from: monit@foo.bar }
+#
+#
+## You can set the alert recipients here, which will receive the alert for
+## each service. The event alerts may be restricted using the list.
+#
+# set alert sysadm@foo.bar # receive all alerts
+# set alert manager@foo.bar only on { timeout } # receive just service-
+# # timeout alert
+
+# excluding instance, changed
+set alert <%= rubber_env.admin_email %> only on {
+ connection
+ checksum
+ data
+ exec
+ gid
+ icmp
+ invalid
+ nonexist
+ permission
+ resource
+ size
+ timeout
+ timestamp
+ uid
+}
+
+#
+#
+## Monit has an embedded webserver, which can be used to view the
+## configuration, actual services parameters or manage the services using the
+## web interface.
+#
+# set httpd port 2812 and
+# use address localhost # only accept connection from localhost
+# allow localhost # allow localhost to connect to the server and
+# allow admin:monit # require user 'admin' with password 'monit'
+#
+
+<%
+ tools_server = rubber_instances.for_role('web_tools').first.internal_ip rescue "127.0.0.1"
+%>
+set httpd port <%= rubber_env.monit_admin_port %>
+ allow <%= tools_server %>
+
+#
+###############################################################################
+## Services
+###############################################################################
+##
+## Check the general system resources such as load average, cpu and memory
+## usage. Each rule specifies the tested resource, the limit and the action
+## which will be performed in the case that the test failed.
+#
+# check system myhost.mydomain.tld
+# if loadavg (1min) > 4 then alert
+# if loadavg (5min) > 2 then alert
+# if memory usage > 75% then alert
+# if cpu usage (user) > 70% then alert
+# if cpu usage (system) > 30% then alert
+# if cpu usage (wait) > 20% then alert
+#
+#
+## Check a file for existence, checksum, permissions, uid and gid. In addition
+## to the recipients in the global section, customized alert will be send to
+## the additional recipient. The service may be grouped using the GROUP option.
+#
+# check file apache_bin with path /usr/local/apache/bin/httpd
+# if failed checksum and