Permalink
Browse files

Initial commit.

  • Loading branch information...
0 parents commit 144b16c899aa1d55e624621a13d72138b58f162d @tweibley tweibley committed Dec 10, 2012
Showing with 219 additions and 0 deletions.
  1. +66 −0 README.md
  2. +73 −0 intermission.lua
  3. +44 −0 intermission_helpers.lua
  4. +36 −0 sample-nginx.conf
@@ -0,0 +1,66 @@
+#intermission
+
+
+intermission is a bit of [OpenResty](http://openresty.org) magic written in Lua to help you perform zero down time maintenance. At [37signals](http://37signals.com) we use this to perform database maintenance with limited impact to the user (especially when combined with [mysql\_role\_swap](https://github.com/37signals/mysql_role_swap/).
+
+## Design Concepts:
+Put an incoming web request on hold long enough to do bad things behind the scenes. Release the incoming requests in the same order they were received. Have limited dependencies ([redis](http://redis.io)).
+
+## Improvements:
++ The current use of redis lists makes requests vulnerable to being forever paused. We can either add a global timeout or do some other magic to make item removal from the lists less vulnerable.
++ We could also abandon the use of redis and just track things on each local Nginx instance.
+
+## Gotchas:
+Requests can only be paused as long the device sitting in front of it will allow. If you have [haproxy](haproxy.1wt.eu) deployed in front of your Nginx instance, make sure to check your "srvtimeout" values.
+
+# Getting started
+### Installation
+
++ Install [OpenResty](http://openresty.org) and compile it with the [nginx-x-rid-header](https://github.com/newobj/nginx-x-rid-header) module.
++ Make sure you add the [lua-resty-redis](https://github.com/agentzh/lua-resty-redis) module in the right spot if it's not already there.
++ Install and setup a [redis](http://redis.io) instance.
+
+### Trying it Out
+
++ Run redis-server
++ Copy all of the files (except this one!) to /usr/local/openresty/nginx/conf (assuming a default installation).
++ Run /usr/local/openresty/nginx/sbin/nginx -c conf/sample-nginx.conf.
++ Tail /usr/local/openresty/nginx/log/intermission-error.log.
++ You might also use redis-cli with the monitor command.
++ Hit [http://localhost:8080](http://localhost:8080) (you should see google)
++ Hit [http://localhost:8080/control](http://localhost:8080/control).
++ Hit [http://localhost:8080](http://localhost:8080) (you should see nothing)
++ Hit [http://localhost:8080/control](http://localhost:8080/control).
++ (Your request should have gone through now...)
+
+# Getting help and contributing
+
+### Getting help with intermission
+The fastest way to get help is to send an email to intermission@librelist.com.
+Github issues and pull requests are checked regularly.
+
+### Contributing
+Pull requests with passing tests (there are no tests!) are welcomed and appreciated.
+
+# License
+
+ Copyright (c) 2012 37signals (37signals.com)
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,73 @@
+--Copyright 2012 37signals / Taylor Weibley
+local x_pause_id = ngx.var.x_request_id;
+local sleep_time = ngx.var.intermission_interval;
+ngx.log(ngx.ERR, "Redis IP: " .. ngx.var.redis_ip .. " and Port: " .. ngx.var.redis_port)
+
+local redis = require "resty.redis"
+local redis_key_prefix = ngx.var.redis_key_prefix;
+local red = redis:new()
+red:set_timeout(ngx.var.x_redis_timeout)
+
+-- Try to connect to redis and log an error if that fails and pass the request through.
+local ok, err = red:connect(ngx.var.redis_ip, ngx.var.redis_port)
+if not ok then
+ ngx.log(ngx.ERR, "Failed to connect to redis: " .. err)
+ return
+end
+
+-- Ask redis for the pause key, if it's not there pass the request through.
+local res, err = red:get(redis_key_prefix .. 'pause')
+if not res then
+ ngx.log(ngx.ERR, "Failed to get pause key: " .. err)
+ return
+end
+
+-- We've got a result and we need to either pass the request (not paused) or hold on to it (paused).
+if res == ngx.null then -- not in pause mode
+ -- Put it into the connection pool of size 5, with 0 idle timeout.
+ local ok, err = red:set_keepalive(0, 5)
+ if not ok then
+ ngx.log(ngx.ERR, "Failed to set keepalive: " .. err)
+ end
+ return
+
+else
+ -- We are pausing...
+ ngx.log(ngx.ERR, "Entering pause mode: " .. x_pause_id)
+
+ local count = 0;
+ while true do
+ -- On the first loop add our req_id to the list.
+ if count == 0 then
+ local ok, err = red:lpush(redis_key_prefix .. 'paused', x_pause_id)
+ if not ok then
+ ngx.log(ngx.ERR, "failed to lpush: " .. err)
+ end
+ end
+
+ -- It is possible we are already unpaused, so lets check.
+ local res, err = red:get(redis_key_prefix .. 'pause')
+ if res == ngx.null then
+ local val, err = red:lindex(redis_key_prefix .. 'paused', -1)
+ if val == x_pause_id then
+ ngx.log(ngx.ERR, "Ready to exit pause because req_id " .. x_pause_id .. " is last on list: " .. val)
+ local ok, err = red:rpop(redis_key_prefix .. 'paused') -- remove ourself from the list
+ -- If the operation fails, we need to bail out
+ if not ok then
+ ngx.log(ngx.ERR, "failed to rpop: " .. err)
+ return
+ end
+ ngx.log(ngx.ERR, "Exiting pause mode after " .. count .. " seconds" .. " for req_id " .. val)
+ local ok, err = red:set_keepalive(0, 5)
+ if not ok then
+ ngx.log(ngx.ERR, "Failed to set keepalive: " .. err)
+ return
+ end
+ break
+ end
+ end
+ -- Pause for however long and update the count of seconds.
+ ngx.sleep(sleep_time)
+ count = count + sleep_time;
+ end
+end
@@ -0,0 +1,44 @@
+--Copyright 2012 37signals / Taylor Weibley
+
+local redis = require "resty.redis"
+local redis_key_prefix = ngx.var.redis_key_prefix;
+local red = redis:new()
+red:set_timeout(ngx.var.x_redis_timeout)
+
+
+-- Try to connect to redis and log an error if that fails and pass the request through.
+local ok, err = red:connect(ngx.var.redis_ip, ngx.var.redis_port)
+if not ok then
+ ngx.log(ngx.ERR, "Failed to connect to redis: " .. err)
+ ngx.say("Unable to connect to redis!")
+ ngx.exit(500)
+end
+
+-- Ask redis for the pause key, if it's not there pass the request through.
+local res, err = red:get(redis_key_prefix .. 'pause')
+if not res then
+ ngx.log(ngx.ERR, "Failed to get pause key: " .. err)
+ ngx.say("Unable to get pause key!")
+ ngx.exit(500)
+end
+if res == ngx.null then -- not in pause mode
+ local res, err = red:set(redis_key_prefix .. 'pause', 1)
+ if not res then
+ ngx.log(ngx.ERR, "Failed to set pause key: " .. err)
+ ngx.say("Unable to set pause key!")
+ ngx.exit(500)
+ else
+ ngx.say("Request paused!")
+ ngx.exit(200)
+ end
+else
+ local res, err = red:del(redis_key_prefix .. 'pause')
+ if not res then
+ ngx.log(ngx.ERR, "Failed to remove pause key: " .. err)
+ ngx.say("Unable to delete pause key!")
+ ngx.exit(500)
+ else
+ ngx.say("Request no longer paused!")
+ ngx.exit(200)
+ end
+end
@@ -0,0 +1,36 @@
+worker_processes 2;
+
+error_log logs/error.log debug;
+pid logs/nginx.pid;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+
+ server {
+ listen 8080;
+ server_name localhost;
+
+ location / {
+ error_log logs/intermission-error.log debug;
+ set $redis_ip '127.0.0.1';
+ set $redis_port 6379;
+ set $redis_timeout 500; #half a second
+ set $redis_key_prefix 'intermission_test';
+ set $intermission_interval 1; #one second
+ set $x_request_id $request_id;
+ access_by_lua_file conf/intermission.lua;
+ proxy_pass http://google.com;
+ }
+ location /control {
+ error_log logs/intermission-error.log debug;
+ set $redis_ip '127.0.0.1';
+ set $redis_port 6379;
+ set $redis_timeout 500; #half a second
+ set $redis_key_prefix 'intermission_test';
+ content_by_lua_file conf/intermission_helpers.lua;
+ }
+ }
+}

0 comments on commit 144b16c

Please sign in to comment.