Permalink
Browse files

CloudCrowd 0.4.1, with multiple-nodes-per-machine, if desired.

  • Loading branch information...
1 parent 0b69c9e commit d787d9775ea90811cdb64d48918e1040ac85d3db @jashkenas jashkenas committed Apr 22, 2010
View
4 cloud-crowd.gemspec
@@ -1,7 +1,7 @@
Gem::Specification.new do |s|
s.name = 'cloud-crowd'
- s.version = '0.4.0' # Keep version in sync with cloud-cloud.rb
- s.date = '2010-03-31'
+ s.version = '0.4.1' # Keep version in sync with cloud-cloud.rb
+ s.date = '2010-04-22'
s.homepage = "http://wiki.github.com/documentcloud/cloud-crowd"
s.summary = "Parallel Processing for the Rest of Us"
View
2 lib/cloud-crowd.rb
@@ -44,7 +44,7 @@ module CloudCrowd
autoload :WorkUnit, 'cloud_crowd/models'
# Keep this version in sync with the gemspec.
- VERSION = '0.4.0'
+ VERSION = '0.4.1'
# Increment the schema version when there's a backwards incompatible change.
SCHEMA_VERSION = 3
View
3 lib/cloud_crowd/models/job.rb
@@ -48,6 +48,7 @@ def check_for_completion
return queue_for_workers([outs]) if merging?
if complete?
update_attributes(:outputs => outs, :time => time_taken)
+ puts "Job ##{id} (#{action}) #{display_status}."
Thread.new { fire_callback } if callback_url
end
self
@@ -76,7 +77,7 @@ def fire_callback
response = RestClient.post(callback_url, {:job => self.to_json})
Thread.new { self.destroy } if response && response.code == 201
rescue RestClient::Exception => e
- puts "Job ##{id} failed to fire callback: #{callback_url}"
+ puts "Job ##{id} (#{action}) failed to fire callback: #{callback_url}"
end
end
View
7 lib/cloud_crowd/models/node_record.rb
@@ -17,13 +17,16 @@ class NodeRecord < ActiveRecord::Base
:order => 'updated_at asc'
}
+ # Extract the port number from the host id.
+ PORT = /:(\d+)\Z/
+
# Register a Node with the central server. This happens periodically
# (once every `Node::CHECK_IN_INTERVAL` seconds). Nodes will be de-registered
# if they checked in within a reasonable interval.
def self.check_in(params, request)
attrs = {
:ip_address => request.ip,
- :port => params[:port],
+ :port => params[:host].match(PORT)[1].to_i,
:busy => params[:busy],
:max_workers => params[:max_workers],
:enabled_actions => params[:enabled_actions]
@@ -62,7 +65,7 @@ def busy?
# The URL at which this Node may be reached.
# TODO: Make sure that the host actually has externally accessible DNS.
def url
- @url ||= "http://#{host}:#{port}"
+ @url ||= "http://#{host}"
end
# Keep a RestClient::Resource handy for contacting the Node, including
View
6 lib/cloud_crowd/node.rb
@@ -70,6 +70,7 @@ def initialize(port=nil, daemon=false)
@host = Socket.gethostname
@enabled_actions = CloudCrowd.actions.keys - (CloudCrowd.config[:disabled_actions] || [])
@port = port || DEFAULT_PORT
+ @id = "#{@host}:#{@port}"
@daemon = daemon
@overloaded = false
@max_load = CloudCrowd.config[:max_load]
@@ -99,8 +100,7 @@ def start
# configuration of this Node. If it can't check-in, there's no point in
# starting.
def check_in(critical=false)
- @central["/node/#{@host}"].put(
- :port => @port,
+ @central["/node/#{@id}"].put(
:busy => @overloaded,
:max_workers => CloudCrowd.config[:max_workers],
:enabled_actions => @enabled_actions.join(',')
@@ -113,7 +113,7 @@ def check_in(critical=false)
# Before exiting, the Node checks out with the central server, releasing all
# of its WorkUnits for other Nodes to handle
def check_out
- @central["/node/#{@host}"].delete
+ @central["/node/#{@id}"].delete
end
# Lazy-initialize the asset_store, preferably after the Node has launched.
View
3 lib/cloud_crowd/server.rb
@@ -71,6 +71,7 @@ class Server < Sinatra::Base
post '/jobs' do
job = Job.create_from_request(JSON.parse(params[:job]))
WorkUnit.distribute_to_nodes
+ puts "Job ##{job.id} (#{job.action}) started."
json job
end
@@ -94,13 +95,15 @@ class Server < Sinatra::Base
put '/node/:host' do
NodeRecord.check_in(params, request)
WorkUnit.distribute_to_nodes
+ puts "Node #{params[:host]} checked in."
json nil
end
# Deregisters a Node from the central server. Releases and redistributes any
# WorkUnits it may have had checked out.
delete '/node/:host' do
NodeRecord.destroy_all(:host => params[:host])
+ puts "Node #{params[:host]} checked out."
json nil
end
View
6 wiki/change_log.textile
@@ -1,3 +1,9 @@
+h3. Version 0.4.1
+
+* Making the canonical identifier for nodes the `host:port`, so that it's
+possible (although still undesirable) to have multiple nodes running different
+actions on the same physical machine.
+
h3. Version 0.4.0
* Updated the Operations Center UI for workflows with large numbers of jobs -- individual jobs are no longer displayed, only aggregate counts. There's a link to grab a quick tail of the last 100 lines of your server log, for debugging.

0 comments on commit d787d97

Please sign in to comment.