Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

added tags to CloudCrowd nodes.

  • Loading branch information...
commit 6a73424acc32487541a7a8d48593d9a5983230c3 1 parent f9ce039
@jashkenas jashkenas authored
View
2  lib/cloud-crowd.rb
@@ -47,7 +47,7 @@ module CloudCrowd
VERSION = '0.4.1'
# Increment the schema version when there's a backwards incompatible change.
- SCHEMA_VERSION = 3
+ SCHEMA_VERSION = 4
# Root directory of the CloudCrowd gem.
ROOT = File.expand_path(File.dirname(__FILE__) + '/..')
View
67 lib/cloud_crowd/command_line.rb
@@ -2,13 +2,13 @@
module CloudCrowd
class CommandLine
-
+
# Configuration files required for the `crowd` command to function.
CONFIG_FILES = ['config.yml', 'config.ru', 'database.yml']
-
+
# Reference the absolute path to the root.
CC_ROOT = File.expand_path(File.dirname(__FILE__) + '/../..')
-
+
# Command-line banner for the usage message.
BANNER = <<-EOS
CloudCrowd is a MapReduce-inspired Parallel Processing System for Ruby.
@@ -25,13 +25,13 @@ class CommandLine
console Launch a CloudCrowd console, connected to the central database
load_schema Load the schema into the database specified by database.yml
cleanup Removes jobs that were finished over --days (7 by default) ago
-
+
server -d [start | stop | restart] Servers and nodes can be launched as
node -d [start | stop | restart] daemons, then stopped or restarted.
Options:
EOS
-
+
# Creating a CloudCrowd::CommandLine runs from the contents of ARGV.
def initialize
parse_options
@@ -47,7 +47,7 @@ def initialize
else usage
end
end
-
+
# Spin up an IRB session with the CloudCrowd code loaded in, and a database
# connection established. The equivalent of Rails' `script/console`.
def run_console
@@ -60,7 +60,7 @@ def run_console
Object.send(:include, CloudCrowd)
IRB.start
end
-
+
# `crowd server` can either 'start', 'stop', or 'restart'.
def run_server(subcommand)
load_code
@@ -71,7 +71,7 @@ def run_server(subcommand)
when 'restart' then restart_server
end
end
-
+
# Convenience command for quickly spinning up the central server. More
# sophisticated deployments, load-balancing across multiple app servers,
# should use the config.ru rackup file directly. This method will start
@@ -86,19 +86,19 @@ def start_server
puts "Starting CloudCrowd Central Server on port #{port}..."
exec "thin -e #{@options[:environment]} -p #{port} #{daemonize} --tag cloud-crowd-server --log #{log_path} --pid #{pid_path} -R #{rackup_path} start"
end
-
+
# Stop the daemonized central server, if it exists.
def stop_server
Thin::Server.kill(CloudCrowd.pid_path('server.pid'), 0)
end
-
+
# Restart the daemonized central server.
def restart_server
stop_server
sleep 1
start_server
end
-
+
# `crowd node` can either 'start', 'stop', or 'restart'.
def run_node(subcommand)
load_code
@@ -109,34 +109,34 @@ def run_node(subcommand)
when 'restart' then restart_node
end
end
-
+
# Launch a Node. Please only run a single node per machine. The Node process
# will be long-lived, although its workers will come and go.
def start_node
- port = @options[:port] || Node::DEFAULT_PORT
- puts "Starting CloudCrowd Node on port #{port}..."
- Node.new(port, @options[:daemonize])
+ @options[:port] ||= Node::DEFAULT_PORT
+ puts "Starting CloudCrowd Node on port #{@options[:port]}..."
+ Node.new(@options)
end
-
+
# If the daemonized Node is running, stop it.
def stop_node
Thin::Server.kill CloudCrowd.pid_path('node.pid')
end
-
+
# Restart the daemonized Node, if it exists.
def restart_node
stop_node
sleep 1
start_node
end
-
+
# Load in the database schema to the database specified in 'database.yml'.
def run_load_schema
load_code
connect_to_database(false)
require 'cloud_crowd/schema.rb'
end
-
+
# Install the required CloudCrowd configuration files into the specified
# directory, or the current one.
def run_install(install_path)
@@ -148,22 +148,22 @@ def run_install(install_path)
install_file "#{CC_ROOT}/config/database.example.yml", "#{install_path}/database.yml"
install_file "#{CC_ROOT}/actions", "#{install_path}/actions", true
end
-
+
# Clean up all Jobs in the CloudCrowd database older than --days old.
def run_cleanup
load_code
connect_to_database(true)
Job.cleanup_all(:days => @options[:days])
end
-
+
# Print `crowd` usage.
def usage
puts "\n#{@option_parser}\n"
end
-
-
+
+
private
-
+
# Check for configuration files, either in the current directory, or in
# the CLOUD_CROWD_CONFIG environment variable. Exit if they're not found.
def ensure_config
@@ -171,9 +171,9 @@ def ensure_config
found = CONFIG_FILES.all? {|f| File.exists? "#{@options[:config_path]}/#{f}" }
found ? @config_dir = true : config_not_found
end
-
+
# Parse all options for all commands.
- # Valid options are: --config --port --environment --daemonize --days.
+ # Valid options are: --config --port --environment --tag --daemonize --days.
def parse_options
@options = {
:environment => 'production',
@@ -190,6 +190,9 @@ def parse_options
opts.on('-e', '--environment ENV', 'server environment (defaults to production)') do |env|
@options[:environment] = env
end
+ opts.on('-t', '--tag TAG', 'tag a node with a name') do |tag|
+ @options[:tag] = tag
+ end
opts.on('-d', '--daemonize', 'run as a background daemon') do |daemonize|
@options[:daemonize] = daemonize
end
@@ -205,7 +208,7 @@ def parse_options
@option_parser.banner = BANNER
@option_parser.parse!(ARGV)
end
-
+
# Load in the CloudCrowd module code, dependencies, lib files and models.
# Not all commands require this.
def load_code
@@ -213,21 +216,21 @@ def load_code
require "#{CC_ROOT}/lib/cloud-crowd"
CloudCrowd.configure("#{@options[:config_path]}/config.yml")
end
-
+
# Establish a connection to the central server's database. Not all commands
# require this.
def connect_to_database(validate_schema)
require 'cloud_crowd/models'
CloudCrowd.configure_database("#{@options[:config_path]}/database.yml", validate_schema)
end
-
+
# Exit with an explanation if the configuration files couldn't be found.
def config_not_found
puts "`crowd` can't find the CloudCrowd configuration directory. Please use `crowd -c path/to/config`, or run `crowd` from inside of the configuration directory itself."
exit(1)
end
-
- # Install a file and log the installation. If we're overwriting a file,
+
+ # Install a file and log the installation. If we're overwriting a file,
# offer a chance to back out.
def install_file(source, dest, is_dir=false)
if File.exists?(dest)
@@ -237,6 +240,6 @@ def install_file(source, dest, is_dir=false)
is_dir ? FileUtils.cp_r(source, dest) : FileUtils.cp(source, dest)
puts "installed #{dest}" unless ENV['RACK_ENV'] == 'test'
end
-
+
end
end
View
4 lib/cloud_crowd/models/node_record.rb
@@ -28,6 +28,7 @@ def self.check_in(params, request)
:ip_address => request.ip,
:port => params[:host].match(PORT)[1].to_i,
:busy => params[:busy],
+ :tag => params[:tag],
:max_workers => params[:max_workers],
:enabled_actions => params[:enabled_actions]
}
@@ -93,7 +94,8 @@ def release_work_units
def to_json(opts={})
{ 'host' => host,
'workers' => worker_pids,
- 'status' => display_status
+ 'status' => display_status,
+ 'tag' => tag
}.to_json
end
View
8 lib/cloud_crowd/node.rb
@@ -63,15 +63,16 @@ class Node < Sinatra::Base
end
# When creating a node, specify the port it should run on.
- def initialize(port=nil, daemon=false)
+ def initialize(options={})
require 'json'
CloudCrowd.identity = :node
@central = CloudCrowd.central_server
@host = Socket.gethostname
@enabled_actions = CloudCrowd.actions.keys - (CloudCrowd.config[:disabled_actions] || [])
- @port = port || DEFAULT_PORT
+ @port = options[:port] || DEFAULT_PORT
@id = "#{@host}:#{@port}"
- @daemon = daemon
+ @daemon = !!options[:daemonize]
+ @tag = options[:tag]
@overloaded = false
@max_load = CloudCrowd.config[:max_load]
@min_memory = CloudCrowd.config[:min_free_memory]
@@ -102,6 +103,7 @@ def start
def check_in(critical=false)
@central["/node/#{@id}"].put(
:busy => @overloaded,
+ :tag => @tag,
:max_workers => CloudCrowd.config[:max_workers],
:enabled_actions => @enabled_actions.join(',')
)
View
1  lib/cloud_crowd/schema.rb
@@ -20,6 +20,7 @@
t.integer "port", :null => false
t.string "enabled_actions", :default => '', :null => false
t.boolean "busy", :default => false, :null => false
+ t.string "tag"
t.integer "max_workers"
t.datetime "created_at"
t.datetime "updated_at"
View
3  public/css/admin_console.css
@@ -126,8 +126,9 @@ body {
}
#nodes .node {
font-size: 11px;
- line-height: 22px;
+ line-height: 22px; height: 22px;
background-image: url(../images/server.png);
+ overflow: hidden;
}
#nodes .node.busy {
background-image: url(../images/server_busy.png);
View
7 public/js/admin_console.js
@@ -107,10 +107,11 @@ window.Console = {
$('.has_nodes', header).html(nc + " Node" + (nc != 1 ? 's' : '') + " / " + wc + " Worker" + (wc != 1 ? 's' : ''));
header.toggleClass('no_nodes', this._nodes.length <= 0);
$('#nodes').html($.map(this._nodes, function(node) {
- var html = "";
+ var html = "";
var extra = node.status == 'busy' ? ' <span class="busy">[busy]</span>' : '';
- html += '<div class="node ' + node.status + '">' + node.host + extra + '</div>';
- html += $.map(node.workers, function(pid) {
+ var tag = node.tag ? '[' + node.tag + '] ' : '';
+ html += '<div class="node ' + node.status + '">' + tag + node.host + extra + '</div>';
+ html += $.map(node.workers, function(pid) {
var name = pid + '@' + node.host;
return '<div class="worker" rel="' + name + '">' + name + '</div>';
}).join('');
View
18 test/unit/test_node.rb
@@ -1,17 +1,17 @@
require 'test_helper'
class NodeUnitTest < Test::Unit::TestCase
-
+
context "A Node" do
-
+
setup do
- @node = Node.new(11011).instance_variable_get(:@app)
+ @node = Node.new(:port => 11011).instance_variable_get(:@app)
end
-
+
should "set the identity of the Ruby instance" do
assert CloudCrowd.node?
end
-
+
should "instantiate correctly" do
assert @node.central.to_s == "http://localhost:9173"
assert @node.port == 11011
@@ -19,13 +19,13 @@ class NodeUnitTest < Test::Unit::TestCase
assert @node.enabled_actions.length > 2
assert @node.asset_store.is_a? AssetStore::FilesystemStore
end
-
+
should "trap signals and launch a server at start" do
Thin::Server.any_instance.expects(:start)
@node.expects(:check_in)
@node.start
end
-
+
should "be able to determine if the node is overloaded" do
assert !@node.overloaded?
@node.instance_variable_set :@max_load, 0.01
@@ -35,7 +35,7 @@ class NodeUnitTest < Test::Unit::TestCase
@node.instance_variable_set :@min_memory, 8000
assert @node.overloaded?
end
-
+
end
-
+
end
Please sign in to comment.
Something went wrong with that request. Please try again.