Permalink
Browse files

Added the chef configs used for the site.

  • Loading branch information...
1 parent 5bf8788 commit 23342973f79c86d2c35e7383ed3c4820e8bac02c @ericholscher ericholscher committed Apr 3, 2011
Showing with 800 additions and 0 deletions.
  1. +52 −0 deploy/cookbooks/main/definitions/virtualenv.rb
  2. +15 −0 deploy/cookbooks/main/files/default/bash_profile
  3. +12 −0 deploy/cookbooks/main/files/default/celery.conf
  4. +14 −0 deploy/cookbooks/main/files/default/gunicorn.conf
  5. +24 −0 deploy/cookbooks/main/files/default/memcached.conf
  6. +24 −0 deploy/cookbooks/main/files/default/nginx.conf
  7. +61 −0 deploy/cookbooks/main/files/default/readthedocs.vcl
  8. +12 −0 deploy/cookbooks/main/files/default/varnish.conf
  9. +12 −0 deploy/cookbooks/main/files/default/varnishlog.conf
  10. +15 −0 deploy/cookbooks/main/recipes/celery.rb
  11. +66 −0 deploy/cookbooks/main/recipes/default.rb
  12. +17 −0 deploy/cookbooks/main/recipes/gunicorn.rb
  13. +18 −0 deploy/cookbooks/main/recipes/memcached.rb
  14. +21 −0 deploy/cookbooks/main/recipes/munin.rb
  15. +31 −0 deploy/cookbooks/main/recipes/nginx.rb
  16. +19 −0 deploy/cookbooks/main/recipes/python.rb
  17. +66 −0 deploy/cookbooks/main/recipes/readthedocs.rb
  18. +17 −0 deploy/cookbooks/main/recipes/security.rb
  19. +40 −0 deploy/cookbooks/main/recipes/varnish.rb
  20. +7 −0 deploy/cookbooks/main/templates/default/hosts
  21. +27 −0 deploy/cookbooks/main/templates/default/munin-node.conf
  22. +131 −0 deploy/cookbooks/main/templates/default/readthedocs
  23. +31 −0 deploy/fabfile.py
  24. +8 −0 deploy/node.json
  25. +6 −0 deploy/roles/asgard.json
  26. +8 −0 deploy/roles/build.json
  27. +28 −0 deploy/roles/default.json
  28. +6 −0 deploy/roles/varnish.json
  29. +8 −0 deploy/roles/web.json
  30. +4 −0 deploy/solo.rb
@@ -0,0 +1,52 @@
+#VIA: https://gist.github.com/612395
+
+# Definition to create virtualenvs
+#
+# For example::
+#
+# virtualenv "/home/me/myenv" do
+# packages "Django" => "1.2.3"
+# end
+#
+# This would create a new virtualenv in /home/me/myenv and install
+# Django 1.2.3. "packages" is a hash, so you can include multiple
+# packages there. Right now there's nothing to say "latest version"
+# because I don't know Ruby that well!
+#
+# The definition also accepts path, owner, group, and mode arguments, just
+# like the directory resource.
+#
+# TODO: support a requirements file.
+#
+
+define :virtualenv, :action => :create, :owner => "root", :group => "root", :mode => 0755, :packages => {} do
+ path = params[:path] ? params[:path] : params[:name]
+ if params[:action] == :create
+ # Manage the directory.
+ directory path do
+ owner params[:owner]
+ group params[:group]
+ mode params[:mode]
+ end
+ execute "create-virtualenv-#{path}" do
+ user params[:owner]
+ group params[:group]
+ command "virtualenv #{path}"
+ not_if "test -f #{path}/bin/python"
+ end
+ params[:packages].each_pair do |package, version|
+ pip = "#{path}/bin/pip"
+ execute "install-#{package}-#{path}" do
+ user params[:owner]
+ group params[:group]
+ command "#{pip} install #{package}==#{version}"
+ not_if "[ `#{pip} freeze | grep #{package} | cut -d'=' -f3` = '#{version}' ]"
+ end
+ end
+ elsif params[:action] == :delete
+ directory path do
+ action :delete
+ recursive true
+ end
+ end
+end
@@ -0,0 +1,15 @@
+. .bashrc
+
+export PIP_DOWNLOAD_CACHE=/tmp/pip
+export DJANGO_SETTINGS_MODULE=settings
+export PYTHONPATH=$PYTHONPATH:~/sites/readthedocs.org/checkouts/readthedocs.org/readthedocs
+export EDITOR=vim
+
+. sites/readthedocs.org/bin/activate
+
+cd ~/sites/readthedocs.org/
+
+
+alias chk='cd /home/docs/sites/readthedocs.org/checkouts/readthedocs.org'
+alias run='cd /home/docs/sites/readthedocs.org/run'
+
@@ -0,0 +1,12 @@
+description "Celery for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 20
+
+script
+exec sudo -i -u docs django-admin.py celeryd -f /home/docs/sites/readthedocs.org/run/celery.log -c 3 -E -B
+end script
+
+respawn
@@ -0,0 +1,14 @@
+description "Gunicorn for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 5
+respawn
+
+env VENV="/home/docs/sites/readthedocs.org"
+
+#Serve Gunicorn on localhost, since we run nginx locally as well.
+script
+exec sudo -iu docs $VENV/bin/gunicorn_django --preload -w 2 --log-level debug --log-file $VENV/run/gunicorn.log -p $VENV/run/gunicorn.pid -b 127.0.0.1:8888 $VENV/checkouts/readthedocs.org/readthedocs/settings/postgres.py
+end script
@@ -0,0 +1,24 @@
+# Run memcached as a daemon. This command is implied, and is not needed for the
+# daemon to run. See the README.Debian that comes with this package for more
+# information.
+-d
+
+# Log memcached's output to /var/log/memcached
+logfile /var/log/memcached.log
+
+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
+# Note that the daemon will grow to this size, but does not start out holding this much
+# memory
+-m 64
+
+# Default connection port is 11211
+-p 11211
+
+# Run the daemon as root. The start-memcached will default to running as root if no
+# -u command is present in this config file
+-u nobody
+
+# Specify which IP address to listen on. The default is to listen on all IP addresses
+# This parameter is one of the only security measures that memcached has, so make sure
+# it's listening on a firewalled interface.
+-l 10.177.72.204
@@ -0,0 +1,24 @@
+user www-data;
+worker_processes 1;
+
+error_log /var/log/nginx/error.log;
+pid /var/run/nginx.pid;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ log_format host '$host $remote_addr - $remote_user [$time_local] '
+ '"$request" $status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+ include /etc/nginx/mime.types;
+ access_log /var/log/nginx/access.log;
+ sendfile on;
+ keepalive_timeout 65;
+ tcp_nodelay on;
+ gzip on;
+ gzip_disable "MSIE [1-6]\.(?!.*SV1)";
+ gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+ include /etc/nginx/sites-enabled/*;
+}
@@ -0,0 +1,61 @@
+backend chimera {
+ .host = "10.177.72.204";
+ .port = "8000";
+}
+
+backend ladon {
+ .host = "10.177.73.65";
+ .port = "8000";
+}
+
+director doubleteam round-robin {
+ {
+ .backend = chimera;
+ }
+ # server2
+ {
+ .backend = ladon;
+ }
+}
+
+acl purge {
+ "localhost";
+ "192.0.2.14";
+}
+
+sub vcl_recv {
+ set req.backend = doubleteam;
+ if (req.request == "PURGE") {
+ if (!client.ip ~ purge) {
+ error 405 "Not allowed.";
+ }
+ purge("req.url ~ " req.url " && req.http.host == " req.http.host);
+ error 200 "Purged.";
+ }
+ set req.grace = 2m;
+ if (req.http.host != "readthedocs.org") {
+ unset req.http.Cookie;
+ unset req.http.cache-control;
+ return(lookup);
+ }
+
+ // Remove has_js and Google Analytics cookies.
+ set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(__[a-z]+|has_js)=[^;]*", "");
+ // Remove a ";" prefix, if present.
+ set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
+ // Remove empty cookies.
+ if (req.http.Cookie ~ "^\s*$") {
+ unset req.http.Cookie;
+ }
+ if (req.url ~ "\.(png|gif|jpg|swf|css|js|ico)$") {
+ unset req.http.cookie;
+ }
+}
+
+sub vcl_fetch {
+ set beresp.ttl = 2m;
+ set req.grace = 5m;
+ if (req.http.host != "readthedocs.org") {
+ set beresp.ttl = 10m;
+ }
+}
@@ -0,0 +1,12 @@
+description "Varnish for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 20
+
+script
+exec varnishd -F -f /etc/varnish/readthedocs.vcl
+end script
+
+respawn
@@ -0,0 +1,12 @@
+description "Varnish Logging for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 20
+
+script
+exec varnishncsa -af -w /var/log/varnish/apache.log
+end script
+
+respawn
@@ -0,0 +1,15 @@
+cookbook_file "/etc/init/readthedocs-celery.conf" do
+ source "celery.conf"
+ owner "root"
+ group "root"
+ mode 0644
+ notifies :restart, "service[readthedocs-celery]"
+end
+
+service "readthedocs-celery" do
+ provider Chef::Provider::Service::Upstart
+ enabled true
+ running true
+ supports :restart => true, :reload => true, :status => true
+ action [:enable, :start]
+end
@@ -0,0 +1,66 @@
+#
+# Basic server config: basic users, packages, etc.
+#
+
+### Packages
+# Just base packages required by the whole system here, please. Dependencies
+# for other recipes should live int hose recipes.
+
+node[:base_packages].each do |pkg|
+ package pkg do
+ :upgrade
+ end
+end
+
+### Users/groups
+
+# Does the following setup for each user defined in node.json:
+# - creates a group and user paid with a matching uid/guid
+# - creates the home directory
+# - keys the user using a key from the config.
+#
+# Then creates a group for each group defined in the JSON.
+
+
+if node.attribute?("all_servers")
+ template "/etc/hosts" do
+ source "hosts"
+ mode 644
+ variables :all_servers => node[:all_servers] || {}
+ end
+end
+
+node[:users].each_pair do |username, info|
+ group username do
+ gid info[:id]
+ end
+
+ user username do
+ comment info[:full_name]
+ uid info[:id]
+ gid info[:id]
+ shell info[:disabled] ? "/sbin/nologin" : "/bin/bash"
+ supports :manage_home => true
+ home "/home/#{username}"
+ end
+
+ directory "/home/#{username}/.ssh" do
+ owner username
+ group username
+ mode 0700
+ end
+
+ file "/home/#{username}/.ssh/authorized_keys" do
+ owner username
+ group username
+ mode 0600
+ content info[:key]
+ end
+end
+
+node[:groups].each_pair do |name, info|
+ group name do
+ gid info[:gid]
+ members info[:members]
+ end
+end
@@ -0,0 +1,17 @@
+# Gunicorn setup
+
+cookbook_file "/etc/init/readthedocs-gunicorn.conf" do
+ source "gunicorn.conf"
+ owner "root"
+ group "root"
+ mode 0644
+ notifies :restart, "service[readthedocs-gunicorn]"
+end
+
+service "readthedocs-gunicorn" do
+ provider Chef::Provider::Service::Upstart
+ enabled true
+ running true
+ supports :restart => true, :reload => true, :status => true
+ action [:enable, :start]
+end
@@ -0,0 +1,18 @@
+package "memcached" do
+ :upgrade
+end
+
+service "memcached" do
+ enabled true
+ running true
+ supports :status => true, :restart => true
+ action [:enable, :start]
+end
+
+cookbook_file "/etc/memcached.conf" do
+ source "memcached.conf"
+ mode 0640
+ owner "root"
+ group "root"
+ notifies :restart, resources(:service => "memcached")
+end
@@ -0,0 +1,21 @@
+package "munin-node" do
+ :upgrade
+end
+
+service "munin-node" do
+ enabled true
+ running true
+ supports :status => true, :restart => true, :reload => true
+ action [:enable, :start]
+end
+
+if node.attribute?("munin_servers")
+ template "/etc/munin/munin-node.conf" do
+ source "munin-node.conf"
+ mode 0640
+ owner "root"
+ group "root"
+ variables :munin_servers => node[:munin_servers] || []
+ notifies :restart, resources(:service => "munin-node")
+ end
+end
Oops, something went wrong. Retry.

0 comments on commit 2334297

Please sign in to comment.