Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

First checkin

  • Loading branch information...
commit e9998ed0a994b67c4f0e24bdf48d1ea194e9cc6a 0 parents
@tobi authored
22 README.rdoc
@@ -0,0 +1,22 @@
+= EM-Proxy
+
+EventMachine Proxy DSL:
+- http://www.igvita.com/2009/04/20/ruby-proxies-for-scale-and-monitoring/
+
+== Simple port forwarding proxy
+
+ Proxy.start(:host => "0.0.0.0", :port => 80) do |conn|
+ conn.server :srv, :host => "127.0.0.1", :port => 81
+
+ # modify / process request stream
+ conn.on_data do |data|
+ p [:on_data, data]
+ data
+ end
+
+ # modify / process response stream
+ conn.on_response do |backend, resp|
+ p [:on_response, backend, resp]
+ resp
+ end
+ end
71 api_proxy.rb
@@ -0,0 +1,71 @@
+
+require 'lib/em-proxy'
+require 'memcached'
+require 'httparty'
+require 'zlib'
+
+$cache = Memcached.new('localhost:11211')
+
+class Request
+ Command = /(GET|POST|PUT|HEAD|DELETE) (\/\S*)\r\n/
+ Host = /^(Host\: .*\r\n)/
+
+ attr_accessor :data, :method, :path
+
+ def initialize(data)
+ @data = data
+ _, @method, @path = *Command.scan(data)
+ end
+
+ def add_header(name, value)
+ @data.sub(Host, "#{$1}#{name}: #{value}\r\n")
+ end
+end
+
+def Proxy
+ def initialize(url)
+ @body = url
+ end
+
+ def forward_request
+ Net::HTTP.
+ end
+
+ def crc32
+ Zlib.crc32(@body, 0)
+ end
+end
+
+
+ # /proxy/*
+
+Proxy.start(:host => "0.0.0.0", :port => 3005) do |conn|
+ conn.server :shopify, :host => "127.0.0.1", :port => 80
+
+ # put <pri> <delay> <ttr> <bytes>\r\n
+
+ conn.on_data do |data|
+ request = Request.new(data)
+
+ if request.path =~ /^\/proxy/
+ proxy = Proxy.new(request.request_uri)
+
+ if proxy.available?
+
+ proxy.forward_request
+
+ $cache.set proxy.cache_key, proxy.content
+
+ request.add_header('X-Proxy-Status', proxy.status)
+ request.add_header('X-Proxy-Content', proxy.cache_key)
+ request.data
+ end
+
+ request.data
+ end
+ end
+
+ conn.on_response do |backend, resp|
+ resp
+ end
+end
12 examples/appserver.rb
@@ -0,0 +1,12 @@
+require "rubygems"
+require "rack"
+
+app = lambda {
+ p [:serving, ARGV[0]]
+ r = rand(2)
+
+ sleep(r)
+ [200, {"Content-Type" => "text/plain"}, ["hello world: #{r}"]]
+}
+
+Rack::Handler::Mongrel.run(app, {:Host => "0.0.0.0", :Port => ARGV[0]})
43 examples/beanstalkd_interceptor.rb
@@ -0,0 +1,43 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 11300) do |conn|
+ conn.server :srv, :host => "127.0.0.1", :port => 11301
+
+ # put <pri> <delay> <ttr> <bytes>\r\n
+ PUT_CMD = /put (\d+) (\d+) (\d+) (\d+)\r\n/
+
+ conn.on_data do |data|
+ if put = data.match(PUT_CMD)
+
+ # archive any job > 10 minutes away
+ if put[2].to_i > 600
+ p [:put, :archive]
+ # INSERT INTO ....
+
+ conn.send_data "INSERTED 9999\r\n"
+ data = nil
+ end
+ end
+
+ data
+ end
+
+ conn.on_response do |backend, resp|
+ p [:resp, resp]
+ resp
+ end
+end
+
+#
+# beanstalkd -p 11301 -d
+# ruby examples/beanstalkd_interceptor.rb
+#
+# irb
+# >> require 'beanstalk-client'
+# >> beanstalk = Beanstalk::Pool.new(['127.0.0.1'])
+# >> beanstalk.put("job1")
+# => 1
+# >> beanstalk.put("job2")
+# => 2
+# >> beanstalk.put("job3", 0, 1000)
+# => 9999
36 examples/duplex.rb
@@ -0,0 +1,36 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80) do |conn|
+ @start = Time.now
+ @data = Hash.new("")
+
+ conn.server :prod, :host => "127.0.0.1", :port => 81 # production, will render resposne
+ conn.server :test, :host => "127.0.0.1", :port => 82 # testing, internal only
+
+ conn.on_data do |data|
+ # rewrite User-Agent
+ data.gsub(/User-Agent: .*?\r\n/, 'User-Agent: em-proxy/0.1\r\n')
+ end
+
+ conn.on_response do |server, resp|
+ # only render response from production
+ @data[server] += resp
+ resp if server == :prod
+ end
+
+ conn.on_finish do
+ p [:on_finish, Time.now - @start]
+ p @data
+ end
+end
+
+#
+# ruby examples/appserver.rb 81
+# ruby examples/appserver.rb 82
+# ruby examples/line_interceptor.rb
+# curl localhost
+#
+# > [:on_finish, 1.008561]
+# > {:prod=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 0",
+# :test=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 1"}
+#
22 examples/line_interceptor.rb
@@ -0,0 +1,22 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80) do |conn|
+ conn.server :srv, :host => "127.0.0.1", :port => 81
+
+ conn.on_data do |data|
+ data
+ end
+
+ conn.on_response do |backend, resp|
+ # substitute all mentions of hello to 'good bye', aka intercepting proxy
+ resp.gsub(/hello/, 'good bye')
+ end
+end
+
+#
+# ruby examples/appserver.rb 81
+# ruby examples/line_interceptor.rb
+# curl localhost
+#
+# > good bye world: 0
+#
18 examples/port_forward.rb
@@ -0,0 +1,18 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80) do |conn|
+ conn.server :srv, :host => "127.0.0.1", :port => 81
+
+ # modify / process request stream
+ conn.on_data do |data|
+ p [:on_data, data]
+ data
+ end
+
+ # modify / process response stream
+ conn.on_response do |backend, resp|
+ p [:on_response, backend, resp]
+ # resp = "HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Thu, 30 Apr 2009 03:53:28 GMT\r\nContent-Type: text/plain\r\n\r\nHar!"
+ resp
+ end
+end
107 examples/smtp_spam_filter.rb
@@ -0,0 +1,107 @@
+require 'lib/em-proxy'
+require 'em-http'
+require 'yaml'
+require 'net/http'
+
+Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn|
+ conn.server :srv, :host => "127.0.0.1", :port => 2525
+
+ RCPT_CMD = /RCPT TO:<(.*)?>\r\n/ # RCPT TO:<name@address.com>\r\n
+ FROM_CMD = /MAIL FROM:<(.*)?>\r\n/ # MAIL FROM:<ilya@aiderss.com>\r\n
+ MSG_CMD = /354 Start your message/ # 354 Start your message
+ MSGEND_CMD = /^.\r\n/
+
+ conn.on_data do |data|
+ @from = data.match(FROM_CMD)[1] if data.match(FROM_CMD)
+ @rcpt = data.match(RCPT_CMD)[1] if data.match(RCPT_CMD)
+ @done = true if data.match(MSGEND_CMD)
+
+ if @buffer
+ @msg += data
+ data = nil
+ end
+
+ if @done
+ @buffer = false
+ res = Net::HTTP.post_form(URI.parse('http://api.defensio.com/app/1.2/audit-comment/77ca297d7546705ee2b5136fad0dcaf8.yaml'), {
+ "owner-url" => "http://www.github.com/igrigorik/em-http-request",
+ "user-ip" => "216.16.254.254",
+ "article-date" => "2009/04/01",
+ "comment-author" => @from,
+ "comment-type" => "comment",
+ "comment-content" => @msg})
+
+ defensio = YAML.load(res.body)['defensio-result']
+ p [:defensio, "SPAM: #{defensio['spam']}, Spaminess: #{defensio['spaminess']}"]
+
+ if defensio['spam']
+ conn.send_data "550 No such user here\n"
+ else
+ data = @msg
+ end
+ end
+
+ data
+ end
+
+ conn.on_response do |server, resp|
+ p [:resp, resp]
+
+ if resp.match(MSG_CMD)
+ @buffer = true
+ @msg = ""
+ end
+
+ resp
+ end
+end
+
+# mailtrap run -p 2525 -f /tmp/mailtrap.log
+# ruby examples/smtp_spam_filter.rb
+#
+# >> require 'net/smtp'
+# >> smtp = Net::SMTP.start("localhost", 2524)
+# >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com"
+
+
+# Protocol trace
+#
+# [:srv, :conn_complete]
+# [:srv, "220 localhost MailTrap ready ESTMP\n"]
+# [:relay_from_backend, :srv, "220 localhost MailTrap ready ESTMP\n"]
+# [:resp, "220 localhost MailTrap ready ESTMP\n"]
+# [:connection, "EHLO localhost.localdomain\r\n"]
+# [:srv, "250-localhost offers just ONE extension my pretty"]
+# [:relay_from_backend, :srv, "250-localhost offers just ONE extension my pretty"]
+# [:resp, "250-localhost offers just ONE extension my pretty"]
+# [:srv, "\n250 HELP\n"]
+# [:relay_from_backend, :srv, "\n250 HELP\n"]
+# [:resp, "\n250 HELP\n"]
+# [:connection, "MAIL FROM:<ilya@aiderss.com>\r\n"]
+# [:srv, "250 OK\n"]
+# [:relay_from_backend, :srv, "250 OK\n"]
+# [:resp, "250 OK\n"]
+# [:connection, "RCPT TO:<ilya@igvita.com>\r\n"]
+# [:srv, "250 OK"]
+# [:relay_from_backend, :srv, "250 OK"]
+# [:resp, "250 OK"]
+# [:srv, "\n"]
+# [:relay_from_backend, :srv, "\n"]
+# [:resp, "\n"]
+# [:connection, "DATA\r\n"]
+# [:srv, "354 Start your message"]
+# [:relay_from_backend, :srv, "354 Start your message"]
+# [:resp, "354 Start your message"]
+# [:srv, "\n"]
+# [:relay_from_backend, :srv, "\n"]
+# [:resp, "\n"]
+# [:connection, "Hello World\r\n"]
+# [:connection, ".\r\n"]
+#
+# [:defensio, "SPAM: false, Spaminess: 0.4"]
+#
+# [:srv, "250 OK\n"]
+# [:relay_from_backend, :srv, "250 OK\n"]
+# [:resp, "250 OK\n"]
+#
+
39 examples/smtp_whitelist.rb
@@ -0,0 +1,39 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn|
+ conn.server :srv, :host => "127.0.0.1", :port => 2525
+
+ # RCPT TO:<name@address.com>\r\n
+ RCPT_CMD = /RCPT TO:<(.*)?>\r\n/
+
+ conn.on_data do |data|
+
+ if rcpt = data.match(RCPT_CMD)
+ if rcpt[1] != "ilya@igvita.com"
+ conn.send_data "550 No such user here\n"
+ data = nil
+ end
+ end
+
+ data
+ end
+
+ conn.on_response do |backend, resp|
+ resp
+ end
+end
+
+
+# mailtrap run -p 2525 -f /tmp/mailtrap.log
+# ruby examples/smtp_whitelist.rb
+#
+# >> require 'net/smtp'
+# >> smtp = Net::SMTP.start("localhost", 2524)
+# >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com"
+# => #<Net::SMTP::Response:0xb7dcff5c @status="250", @string="250 OK\n">
+# >> smtp.finish
+# => #<Net::SMTP::Response:0xb7dcc8d4 @status="221", @string="221 Seeya\n">
+#
+# >> smtp.send_message "Hello World!", "ilya@aiderss.com", "missing_user@igvita.com"
+# => Net::SMTPFatalError: 550 No such user here
+#
8 lib/em-proxy.rb
@@ -0,0 +1,8 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require "rubygems"
+require "eventmachine"
+
+%w[ backend proxy connection ].each do |file|
+ require "em-proxy/#{file}"
+end
36 lib/em-proxy/backend.rb
@@ -0,0 +1,36 @@
+module EventMachine
+ module ProxyServer
+ class Backend < EventMachine::Connection
+ attr_accessor :plexer, :data, :name
+
+ def initialize
+ @connected = EM::DefaultDeferrable.new
+ @data = []
+ end
+
+ def connection_completed
+ p [@name, :conn_complete]
+ @connected.succeed
+ end
+
+ def receive_data(data)
+ p [@name, data]
+ @data.push data
+ @plexer.relay_from_backend(@name, data)
+ end
+
+ # Buffer data until the connection to the backend server
+ # is established and is ready for use
+ def send(data)
+ @connected.callback { send_data data }
+ end
+
+ # Notify upstream plexer that the backend server is done
+ # processing the request
+ def unbind
+ p [@name, :unbind]
+ @plexer.unbind_backend(@name)
+ end
+ end
+ end
+end
63 lib/em-proxy/connection.rb
@@ -0,0 +1,63 @@
+module EventMachine
+ module ProxyServer
+ class Connection < EventMachine::Connection
+
+ ##### Proxy Methods
+ def on_data(&blk); @on_data = blk; end
+ def on_response(&blk); @on_response = blk; end
+ def on_finish(&blk); @on_finish = blk; end
+
+ ##### EventMachine
+ def initialize
+ @servers = {}
+ end
+
+ def receive_data(data)
+ p [:connection, data]
+ data = @on_data.call(data)
+
+ @servers.values.compact.each do |s|
+ s.send_data data unless data.nil?
+ end
+ end
+
+ #
+ # initialize connections to backend servers
+ #
+ def server(name, opts)
+ srv = EventMachine::connect(opts[:host], opts[:port], EventMachine::ProxyServer::Backend) do |c|
+ c.name = name
+ c.plexer = self
+ end
+
+ @servers[name] = srv
+ end
+
+ #
+ # relay data from backend server to client
+ #
+ def relay_from_backend(name, data)
+ p [:relay_from_backend, name, data]
+
+ data = @on_response.call(name, data)
+ send_data data unless data.nil?
+ end
+
+ def unbind
+ # terminate any unfinished connections
+ @servers.values.compact.each do |s|
+ s.close_connection
+ end
+
+ close_connection_after_writing
+ @on_finish.call if @on_finish
+ end
+
+ def unbind_backend(name)
+ @servers[name] = nil
+ close_connection_after_writing if @servers.values.compact.size.zero?
+ end
+
+ end
+ end
+end
20 lib/em-proxy/proxy.rb
@@ -0,0 +1,20 @@
+class Proxy
+
+ def self.start(options, &blk)
+ EM.epoll
+ EM.run do
+
+ trap("TERM") { stop }
+ trap("INT") { stop }
+
+ EventMachine::start_server(options[:host], options[:port], EventMachine::ProxyServer::Connection) do |c|
+ c.instance_eval(&blk)
+ end
+ end
+ end
+
+ def self.stop
+ puts "Terminating ProxyServer"
+ EventMachine.stop
+ end
+end
Please sign in to comment.
Something went wrong with that request. Please try again.