Added the chef configs used for the site.

deploy/cookbooks/main/definitions/virtualenv.rb

@@ -0,0 +1,52 @@
+#VIA: https://gist.github.com/612395
+
+# Definition to create virtualenvs
+#
+# For example::
+#
+# virtualenv "/home/me/myenv" do
+# packages "Django" => "1.2.3"
+# end
+#
+# This would create a new virtualenv in /home/me/myenv and install
+# Django 1.2.3. "packages" is a hash, so you can include multiple
+# packages there. Right now there's nothing to say "latest version"
+# because I don't know Ruby that well!
+#
+# The definition also accepts path, owner, group, and mode arguments, just
+# like the directory resource.
+#
+# TODO: support a requirements file.
+#
+
+define :virtualenv, :action => :create, :owner => "root", :group => "root", :mode => 0755, :packages => {} do
+    path = params[:path] ? params[:path] : params[:name]
+    if params[:action] == :create
+        # Manage the directory.
+        directory path do
+            owner params[:owner]
+            group params[:group]
+            mode params[:mode]
+        end
+        execute "create-virtualenv-#{path}" do
+            user params[:owner]
+            group params[:group]
+            command "virtualenv #{path}"
+            not_if "test -f #{path}/bin/python"
+        end
+        params[:packages].each_pair do |package, version|
+            pip = "#{path}/bin/pip"
+            execute "install-#{package}-#{path}" do
+                user params[:owner]
+                group params[:group]
+                command "#{pip} install #{package}==#{version}"
+                not_if "[ `#{pip} freeze | grep #{package} | cut -d'=' -f3` = '#{version}' ]"
+            end
+        end
+    elsif params[:action] == :delete
+        directory path do
+            action :delete
+            recursive true
+        end
+    end
+end

deploy/cookbooks/main/files/default/bash_profile

@@ -0,0 +1,15 @@
+. .bashrc
+
+export PIP_DOWNLOAD_CACHE=/tmp/pip
+export DJANGO_SETTINGS_MODULE=settings
+export PYTHONPATH=$PYTHONPATH:~/sites/readthedocs.org/checkouts/readthedocs.org/readthedocs
+export EDITOR=vim
+
+. sites/readthedocs.org/bin/activate
+
+cd ~/sites/readthedocs.org/
+
+
+alias chk='cd /home/docs/sites/readthedocs.org/checkouts/readthedocs.org'
+alias run='cd /home/docs/sites/readthedocs.org/run'
+

deploy/cookbooks/main/files/default/celery.conf

@@ -0,0 +1,12 @@
+description "Celery for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 20
+
+script
+exec sudo -i -u docs django-admin.py celeryd -f /home/docs/sites/readthedocs.org/run/celery.log -c 3 -E -B
+end script
+
+respawn

deploy/cookbooks/main/files/default/gunicorn.conf

@@ -0,0 +1,14 @@
+description "Gunicorn for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 5
+respawn
+
+env VENV="/home/docs/sites/readthedocs.org"
+
+#Serve Gunicorn on localhost, since we run nginx locally as well.
+script
+exec sudo -iu docs $VENV/bin/gunicorn_django --preload -w 2 --log-level debug --log-file $VENV/run/gunicorn.log -p $VENV/run/gunicorn.pid -b 127.0.0.1:8888 $VENV/checkouts/readthedocs.org/readthedocs/settings/postgres.py
+end script

deploy/cookbooks/main/files/default/memcached.conf

@@ -0,0 +1,24 @@
+# Run memcached as a daemon. This command is implied, and is not needed for the
+# daemon to run. See the README.Debian that comes with this package for more
+# information.
+-d
+
+# Log memcached's output to /var/log/memcached
+logfile /var/log/memcached.log
+
+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
+# Note that the daemon will grow to this size, but does not start out holding this much
+# memory
+-m 64
+
+# Default connection port is 11211
+-p 11211
+
+# Run the daemon as root. The start-memcached will default to running as root if no
+# -u command is present in this config file
+-u nobody
+
+# Specify which IP address to listen on. The default is to listen on all IP addresses
+# This parameter is one of the only security measures that memcached has, so make sure
+# it's listening on a firewalled interface.
+-l 10.177.72.204

deploy/cookbooks/main/files/default/nginx.conf

@@ -0,0 +1,24 @@
+user www-data;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    log_format host '$host $remote_addr - $remote_user [$time_local] '
+                  '"$request" $status  $body_bytes_sent "$http_referer" '
+                  '"$http_user_agent" "$http_x_forwarded_for"';
+    include       /etc/nginx/mime.types;
+    access_log  /var/log/nginx/access.log;
+    sendfile        on;
+    keepalive_timeout  65;
+    tcp_nodelay        on;
+    gzip  on;
+    gzip_disable "MSIE [1-6]\.(?!.*SV1)";
+    gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+    include /etc/nginx/sites-enabled/*;
+}

deploy/cookbooks/main/files/default/readthedocs.vcl

@@ -0,0 +1,61 @@
+backend chimera {
+    .host = "10.177.72.204";
+    .port = "8000";
+}
+
+backend ladon {
+    .host = "10.177.73.65";
+    .port = "8000";
+}
+
+director doubleteam round-robin {
+      {
+              .backend = chimera;
+      }
+      # server2
+      {
+              .backend = ladon;
+      }
+}
+
+acl purge {
+        "localhost";
+        "192.0.2.14";
+}
+
+sub vcl_recv {
+    set req.backend = doubleteam;
+	if (req.request == "PURGE") {
+		if (!client.ip ~ purge) {
+			error 405 "Not allowed.";
+		}
+		purge("req.url ~ " req.url " && req.http.host == " req.http.host);
+		error 200 "Purged.";
+	}
+  set req.grace = 2m;
+  if (req.http.host != "readthedocs.org") {
+    unset req.http.Cookie;
+    unset req.http.cache-control;
+    return(lookup);
+  }
+
+  // Remove has_js and Google Analytics cookies.
+  set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(__[a-z]+|has_js)=[^;]*", "");
+  // Remove a ";" prefix, if present.
+  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
+  // Remove empty cookies.
+  if (req.http.Cookie ~ "^\s*$") {
+    unset req.http.Cookie;
+  }
+  if (req.url ~ "\.(png|gif|jpg|swf|css|js|ico)$") {
+    unset req.http.cookie;
+  }
+}
+
+sub vcl_fetch {
+  set beresp.ttl = 2m;
+  set req.grace = 5m;
+  if (req.http.host != "readthedocs.org") {
+    set beresp.ttl = 10m;
+  }
+}

deploy/cookbooks/main/files/default/varnish.conf

@@ -0,0 +1,12 @@
+description "Varnish for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 20
+
+script
+exec varnishd -F -f /etc/varnish/readthedocs.vcl
+end script
+
+respawn

deploy/cookbooks/main/files/default/varnishlog.conf

@@ -0,0 +1,12 @@
+description "Varnish Logging for ReadTheDocs"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+#Send KILL after 20 seconds
+kill timeout 20
+
+script
+exec varnishncsa -af -w /var/log/varnish/apache.log
+end script
+
+respawn

deploy/cookbooks/main/recipes/celery.rb

@@ -0,0 +1,15 @@
+cookbook_file "/etc/init/readthedocs-celery.conf" do
+    source "celery.conf"
+    owner "root"
+    group "root"
+    mode 0644
+    notifies :restart, "service[readthedocs-celery]"
+end
+
+service "readthedocs-celery" do
+    provider Chef::Provider::Service::Upstart
+    enabled true
+    running true
+    supports :restart => true, :reload => true, :status => true
+    action [:enable, :start]
+end

deploy/cookbooks/main/recipes/default.rb

@@ -0,0 +1,66 @@
+#
+# Basic server config: basic users, packages, etc.
+#
+
+### Packages
+# Just base packages required by the whole system here, please. Dependencies
+# for other recipes should live int hose recipes.
+
+node[:base_packages].each do |pkg|
+    package pkg do
+        :upgrade
+    end
+end
+
+### Users/groups
+
+# Does the following setup for each user defined in node.json:
+#   - creates a group and user paid with a matching uid/guid
+#   - creates the home directory
+#   - keys the user using a key from the config.
+#
+# Then creates a group for each group defined in the JSON.
+
+
+if node.attribute?("all_servers")
+  template "/etc/hosts" do
+    source "hosts"
+    mode 644
+    variables :all_servers => node[:all_servers] || {}
+  end
+end
+
+node[:users].each_pair do |username, info|
+    group username do
+       gid info[:id]
+    end
+
+    user username do
+        comment info[:full_name]
+        uid info[:id]
+        gid info[:id]
+        shell info[:disabled] ? "/sbin/nologin" : "/bin/bash"
+        supports :manage_home => true
+        home "/home/#{username}"
+    end
+
+    directory "/home/#{username}/.ssh" do
+        owner username
+        group username
+        mode 0700
+    end
+
+    file "/home/#{username}/.ssh/authorized_keys" do
+        owner username
+        group username
+        mode 0600
+        content info[:key]
+    end
+end
+
+node[:groups].each_pair do |name, info|
+    group name do
+        gid info[:gid]
+        members info[:members]
+    end
+end

deploy/cookbooks/main/recipes/gunicorn.rb

@@ -0,0 +1,17 @@
+# Gunicorn setup
+
+cookbook_file "/etc/init/readthedocs-gunicorn.conf" do
+    source "gunicorn.conf"
+    owner "root"
+    group "root"
+    mode 0644
+    notifies :restart, "service[readthedocs-gunicorn]"
+end
+
+service "readthedocs-gunicorn" do
+    provider Chef::Provider::Service::Upstart
+    enabled true
+    running true
+    supports :restart => true, :reload => true, :status => true
+    action [:enable, :start]
+end

deploy/cookbooks/main/recipes/memcached.rb

@@ -0,0 +1,18 @@
+package "memcached" do
+    :upgrade
+end
+
+service "memcached" do
+  enabled true
+  running true
+  supports :status => true, :restart => true
+  action [:enable, :start]
+end
+
+cookbook_file "/etc/memcached.conf" do
+  source "memcached.conf"
+  mode 0640
+  owner "root"
+  group "root"
+  notifies :restart, resources(:service => "memcached")
+end

deploy/cookbooks/main/recipes/munin.rb

@@ -0,0 +1,21 @@
+package "munin-node" do
+    :upgrade
+end
+
+service "munin-node" do
+  enabled true
+  running true
+  supports :status => true, :restart => true, :reload => true
+  action [:enable, :start]
+end
+
+if node.attribute?("munin_servers")
+  template "/etc/munin/munin-node.conf" do
+    source "munin-node.conf"
+    mode 0640
+    owner "root"
+    group "root"
+    variables :munin_servers => node[:munin_servers] || []
+    notifies :restart, resources(:service => "munin-node")
+  end
+end