Skip to content

Commit

Permalink
merge
Browse files Browse the repository at this point in the history
  • Loading branch information
technoweenie committed Nov 18, 2010
2 parents 9042615 + 898e915 commit 4023fb8
Show file tree
Hide file tree
Showing 58 changed files with 12,590 additions and 0 deletions.
1 change: 1 addition & 0 deletions CONTRIBUTORS
Expand Up @@ -44,3 +44,4 @@ Thanks to the following people for making this possible
- Robert Boyd
- Marco Borromeo
- Raine Virta
- Christian Legnitto
48 changes: 48 additions & 0 deletions docs/amqp
@@ -0,0 +1,48 @@
AMQP
========

This service lets you publish push and commit messages to a message broker
(like RabbitMQ) via the AMQP protocol.


Install Notes
-------------

1. Server is the hostname of the broker
2. Port is the port to connect (AMQP default is 5672)
3. Vhost is the vhost to use while publishing (default is '/')
4. Exchange is the exchange to use while publishing
* Note that the exchange is a durable topic exchange
** Future versions may allow you to configure this
5. Username is the user to use when publishing to the exchange
6. Password is the pasword to use when publishing to the exchange
* Note that SSL isn't supported yet

Messages are sent for a push with the following routing key format:
"github.push.#{owner}.#{repo}.#{ref}"
where:
owner = payload['repository']['owner']['name']
repo = payload['repository']['name']
ref = payload['ref_name']

Messages are also sent for each commit in a push, with the following routing
key format:
"github.commit.#{owner}.#{repo}.#{ref}.#{author}"
where:
author = commit['author']['email']
(other fields are the same as above)


Developer Notes
---------------

data
- server
- port
- vhost
- exchange
- username
- password

payload
- refer to docs/github_payload
1 change: 1 addition & 0 deletions github-services.rb
Expand Up @@ -26,6 +26,7 @@
require 'rubyforge'
require 'oauth'
require 'yammer4r'
require 'mq'

set :run, true
set :environment, :production
Expand Down
85 changes: 85 additions & 0 deletions services/amqp.rb
@@ -0,0 +1,85 @@
service :amqp do |data, payload|

EM.run do

# Connect to the AMQP server
connection = AMQP.connect(:host => data['host'] || nil,
:port => data['port'] || 5672,
:user => data['username'] || 'guest',
:pass => data['password'] || 'guest',
:vhost => data['vhost'] || '/',
:logging => false)

if !data['host']
raise GitHub::ServiceConfigurationError, "Invalid server host."
end

if !data['exchange']
raise GitHub::ServiceConfigurationError, "Invalid exchange."
end

# Open a channel on the AMQP connection
channel = MQ.new(connection)

# Create a topic exchange
exchange = MQ::Exchange.new(channel,
:topic,
data['exchange'],
:durable => true)

# Modify the commits a bit
payload['commits'].each do |commit|
commit['files'] = {
'added' => commit['added'],
'modified' => commit['modified'],
'removed' => commit['removed'],
}
commit.delete('added')
commit.delete('modified')
commit.delete('removed')
end

# Generate the push routing key
owner = payload['repository']['owner']['name']
repo = payload['repository']['name']
ref = payload['ref_name']
routing_key = "github.push.#{owner}.#{repo}.#{ref}"

# Assemble the push message
msg = {}
msg['_meta'] = {
'routing_key' => routing_key,
'exchange' => data['exchange'],
}
msg['payload'] = payload

# Publish the push message to the exchange
exchange.publish(msg.to_json,
:key => routing_key,
:content_type => 'application/json')

# Publish individual commit messages
payload['commits'].each do |commit|
# Generate the commit routing key
author = commit['author']['email']
routing_key = "github.commit.#{owner}.#{repo}.#{ref}.#{author}"

# Assemble the commit message
msg = {}
msg['_meta'] = {
'routing_key' => routing_key,
'exchange' => data['exchange'],
}
msg['payload'] = commit

# Publish the commit message to the exchange
exchange.publish(msg.to_json,
:key => routing_key,
:content_type => 'application/json')
end

connection.close{ EM.stop_event_loop }

end

end
128 changes: 128 additions & 0 deletions vendor/amqp-0.6.7/README
@@ -0,0 +1,128 @@
Simple AMQP driver for Ruby/EventMachine
(c) 2008 Aman Gupta (tmm1)

http://github.com/tmm1/amqp
http://rubyforge.org/projects/amqp
http://hopper.squarespace.com/blog/2008/7/22/simple-amqp-library-for-ruby.html
http://groups.google.com/group/ruby-amqp
http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/2008-July/001417.html

This library works with Ruby 1.8, Ruby 1.9, JRuby and Rubinius, and is licensed under the Ruby License.

This library was tested primarily with RabbitMQ, although it should be compatible with any
server implementing the AMQP 0-8 spec.

To use with RabbitMQ, first run the server:

hg clone http://hg.rabbitmq.com/rabbitmq-codegen
hg clone http://hg.rabbitmq.com/rabbitmq-server
cd rabbitmq-server
make run

To get started, refer to the various bundled examples:

ruby examples/mq/pingpong.rb # 1-1 communication using amq.direct
ruby examples/mq/clock.rb # 1-N communication using amq.fanout
ruby examples/mq/stocks.rb # 1-subscriber communication using amq.topic

ruby examples/mq/multiclock.rb # header based routing (new rabbitmq feature)
ruby examples/mq/ack.rb # using ack
ruby examples/mq/pop.rb # pop off messages one at a time

ruby examples/mq/hashtable.rb # simple async rpc layer
ruby examples/mq/primes.rb 4 # parallelized prime number generation
ruby examples/mq/logger.rb # simple logging api

For more details into the lower level AMQP client API, run the simple client example:

ruby examples/amqp/simple.rb # low-level AMQP api
ruby examples/mq/internal.rb # low-level Queue/Exchange api

Or refer to protocol/doc.txt, which enumerates packets sent between a server and client
during a typical session, in both binary and decoded formats.

To run the test suite:

rake spec

The lib/amqp/spec.rb file is generated automatically based on the AMQP specification. To generate it:

rake codegen

This project was inspired by py-amqplib, rabbitmq, qpid and rubbyt.
Special thanks to Dmitriy Samovskiy, Ben Hood and Tony Garnock-Jones.

AMQP resources:

Servers:
RabbitMQ (Rabbit Technologies, Erlang/OTP, MPL) - http://rabbitmq.com
ZeroMQ (iMatix/FastMQ/Intel, C++, GPL3) - http://www.zeromq.org
OpenAMQ (iMatix, C, GPL2) - http://openamq.org
ActiveMQ (Apache Foundation, Java, apache2) - http://activemq.apache.org

Steve Vinoski explains AMQP in his column, Towards Integration
http://steve.vinoski.net/pdf/IEEE-Advanced_Message_Queuing_Protocol.pdf

John O'Hara on the history of AMQP
http://www.acmqueue.org/modules.php?name=Content&pa=showpage&pid=485

Dmitriy's presentation on RabbitMQ/AMQP
http://somic-org.homelinux.org/blog/2008/07/31/slides-for-my-amqprabbitmq-talk/

ZeroMQ's analysis of the messaging technology market
http://www.zeromq.org/whitepapers:market-analysis

Pieter Hintjens's background to AMQP
http://www.openamq.org/doc:amqp-background

Barry Pederson's py-amqplib
http://barryp.org/software/py-amqplib/

Ben Hood on writing an AMQP client
http://hopper.squarespace.com/blog/2008/6/21/build-your-own-amqp-client.html

Dmitriy Samovskiy introduces Ruby + QPid + RabbitMQ
http://somic-org.homelinux.org/blog/2008/06/24/ruby-amqp-rabbitmq-example/

Ben Hood's as3-amqp
http://github.com/0x6e6562/as3-amqp
http://hopper.squarespace.com/blog/2008/7/4/server-side-as3.html
http://hopper.squarespace.com/blog/2008/3/24/as3-amqp-client-first-cut.html

RabbitMQ's protocol code generator
http://hg.rabbitmq.com/rabbitmq-codegen/

Erlang Exchange presentation on the implementation of RabbitMQ
http://skillsmatter.com/podcast/erlang/presenting-rabbitmq-an-erlang-based-implementatio-nof-amqp
http://www.lshift.net/blog/2008/07/01/slides-from-our-erlang-exchange-talk

Jonathan Conway's series on RabbitMQ and using it with Ruby/Merb
http://jaikoo.com/2008/3/20/daemonize-rabbitmq
http://jaikoo.com/2008/3/14/oh-hai-rabbitmq
http://jaikoo.com/2008/2/29/friday-round-up-2008-02-29
http://jaikoo.com/2007/9/4/didn-t-you-get-the-memo

Open Enterprise's series on messaging middleware and AMQP
http://www1.interopsystems.com/analysis/can-amqp-break-ibms-mom-monopoly-part-1.html
http://www1.interopsystems.com/analysis/can-amqp-break-ibms-mom-monopoly-part-2.html
http://www1.interopsystems.com/analysis/can-amqp-break-ibms-mom-monopoly-part-3.html

Messaging and distributed systems resources:

A Critique of the Remote Procedure Call Paradigm
http://www.cs.vu.nl/~ast/publications/euteco-1988.pdf

A Note on Distributed Computing
http://research.sun.com/techrep/1994/smli_tr-94-29.pdf

Convenience Over Correctness
http://steve.vinoski.net/pdf/IEEE-Convenience_Over_Correctness.pdf

Metaprotocol Taxonomy and Communications Patterns
http://hessian.caucho.com/doc/metaprotocol-taxonomy.xtp

Joe Armstrong on Erlang messaging vs RPC
http://armstrongonsoftware.blogspot.com/2008/05/road-we-didnt-go-down.html

SEDA: scalable internet services using message queues
http://www.eecs.harvard.edu/~mdw/papers/seda-sosp01.pdf
15 changes: 15 additions & 0 deletions vendor/amqp-0.6.7/Rakefile
@@ -0,0 +1,15 @@
task :codegen do
sh 'ruby protocol/codegen.rb > lib/amqp/spec.rb'
sh 'ruby lib/amqp/spec.rb'
end

task :spec do
sh 'bacon lib/amqp.rb'
end

task :gem do
sh 'gem build *.gemspec'
end

task :pkg => :gem
task :package => :gem
87 changes: 87 additions & 0 deletions vendor/amqp-0.6.7/amqp.gemspec
@@ -0,0 +1,87 @@
require File.expand_path('../lib/amqp/version', __FILE__)

spec = Gem::Specification.new do |s|
s.name = 'amqp'
s.version = AMQP::VERSION
s.date = '2009-12-29'
s.summary = 'AMQP client implementation in Ruby/EventMachine'
s.email = "amqp@tmm1.net"
s.homepage = "http://amqp.rubyforge.org/"
s.rubyforge_project = 'amqp'
s.description = "An implementation of the AMQP protocol in Ruby/EventMachine for writing clients to the RabbitMQ message broker"
s.has_rdoc = true
s.rdoc_options = '--include=examples'

# ruby -rpp -e' pp `git ls-files`.split("\n").grep(/^(doc|README)/) '
s.extra_rdoc_files = [
"README",
"doc/EXAMPLE_01_PINGPONG",
"doc/EXAMPLE_02_CLOCK",
"doc/EXAMPLE_03_STOCKS",
"doc/EXAMPLE_04_MULTICLOCK",
"doc/EXAMPLE_05_ACK",
"doc/EXAMPLE_05_POP",
"doc/EXAMPLE_06_HASHTABLE"
]

s.authors = ["Aman Gupta"]
s.add_dependency('eventmachine', '>= 0.12.4')

# ruby -rpp -e' pp `git ls-files`.split("\n") '
s.files = [
"README",
"Rakefile",
"amqp.gemspec",
"amqp.todo",
"doc/EXAMPLE_01_PINGPONG",
"doc/EXAMPLE_02_CLOCK",
"doc/EXAMPLE_03_STOCKS",
"doc/EXAMPLE_04_MULTICLOCK",
"doc/EXAMPLE_05_ACK",
"doc/EXAMPLE_05_POP",
"doc/EXAMPLE_06_HASHTABLE",
"examples/amqp/simple.rb",
"examples/mq/ack.rb",
"examples/mq/clock.rb",
"examples/mq/pop.rb",
"examples/mq/hashtable.rb",
"examples/mq/internal.rb",
"examples/mq/logger.rb",
"examples/mq/multiclock.rb",
"examples/mq/pingpong.rb",
"examples/mq/primes-simple.rb",
"examples/mq/primes.rb",
"examples/mq/stocks.rb",
"lib/amqp.rb",
"lib/amqp/version.rb",
"lib/amqp/buffer.rb",
"lib/amqp/client.rb",
"lib/amqp/frame.rb",
"lib/amqp/protocol.rb",
"lib/amqp/server.rb",
"lib/amqp/spec.rb",
"lib/ext/blankslate.rb",
"lib/ext/em.rb",
"lib/ext/emfork.rb",
"lib/mq.rb",
"lib/mq/exchange.rb",
"lib/mq/header.rb",
"lib/mq/logger.rb",
"lib/mq/queue.rb",
"lib/mq/rpc.rb",
"old/README",
"old/Rakefile",
"old/amqp-0.8.json",
"old/amqp_spec.rb",
"old/amqpc.rb",
"old/codegen.rb",
"protocol/amqp-0.8.json",
"protocol/amqp-0.8.xml",
"protocol/codegen.rb",
"protocol/doc.txt",
"research/api.rb",
"research/primes-forked.rb",
"research/primes-processes.rb",
"research/primes-threaded.rb"
]
end

0 comments on commit 4023fb8

Please sign in to comment.