Browse files

14255 - Discovery method should be pluggable

This makes discovery pluggable it includes 2 plugins one thats the
traditional broadcast, one that discovers against a flatfile and a third
has been comitted to the plugins repository that uses the mongodb
discovery plugin to create a local cache

Discovery plugins need DDLs, in the DDL files they declare their
capabilities - can they do fact filter etc. and mcollective will
validate the filter prior to calling the plugin.

RPC clients will force to directed mode for anything but the mc plugin

You'd elect to use a different default discovery mode by setting
default_discovery_mode = flatfile in your client config file

The plugin application will show help for these plugins, it can package
them as rpm/deb and all the usual stuff

The RPC library will adjust discovery time based on the timeout in the
DDL for the plugin unless a custom timeout is specified on the CLI

In a case where compound filters are used we force the 'mc' discovery
plugin since data queries means this is the only thing that makes sense

All application now have a --nodes option that can take a flat file with
a list of nodes
  • Loading branch information...
1 parent 66ed1fe commit d728927b9e981d78ee24cc52e37c7e174f0356c8 @ripienaar ripienaar committed May 30, 2012
Showing with 1,087 additions and 408 deletions.
  1. +7 −2 Rakefile
  2. +22 −0 etc/discovery-help.erb
  3. +2 −9 ext/debian/mcollective-common.install
  4. +3 −7 ext/redhat/mcollective.spec
  5. +15 −7 ext/zsh/_mco
  6. +1 −0 lib/mcollective.rb
  7. +6 −2 lib/mcollective/application.rb
  8. +4 −24 lib/mcollective/client.rb
  9. +8 −2 lib/mcollective/config.rb
  10. +41 −4 lib/mcollective/ddl.rb
  11. +116 −0 lib/mcollective/discovery.rb
  12. +9 −3 lib/mcollective/message.rb
  13. +17 −0 lib/mcollective/optionparser.rb
  14. +65 −19 lib/mcollective/rpc/client.rb
  15. +2 −2 lib/mcollective/rpc/progress.rb
  16. +3 −3 lib/mcollective/runner.rb
  17. +26 −26 lib/mcollective/security/base.rb
  18. +13 −11 lib/mcollective/util.rb
  19. +0 −92 plugins/mcollective/application/controller.rb
  20. +13 −8 plugins/mcollective/application/find.rb
  21. +2 −2 plugins/mcollective/application/plugin.rb
  22. +11 −0 plugins/mcollective/discovery/flatfile.ddl
  23. +40 −0 plugins/mcollective/discovery/flatfile.rb
  24. +11 −0 plugins/mcollective/discovery/mc.ddl
  25. +30 −0 plugins/mcollective/discovery/mc.rb
  26. +1 −0 spec/spec_helper.rb
  27. +18 −18 spec/unit/agents_spec.rb
  28. +4 −2 spec/unit/application_spec.rb
  29. +2 −2 spec/unit/applications_spec.rb
  30. +32 −0 spec/unit/client_spec.rb
  31. +11 −0 spec/unit/config_spec.rb
  32. +0 −8 spec/unit/data/base_spec.rb
  33. +54 −4 spec/unit/ddl_spec.rb
  34. +191 −0 spec/unit/discovery_spec.rb
  35. +2 −2 spec/unit/logger/syslog_logger_spec.rb
  36. +21 −10 spec/unit/message_spec.rb
  37. +2 −2 spec/unit/optionparser_spec.rb
  38. +48 −0 spec/unit/plugins/mcollective/discovery/flatfile_spec.rb
  39. +40 −0 spec/unit/plugins/mcollective/discovery/mc_spec.rb
  40. +2 −2 spec/unit/rpc/actionrunner_spec.rb
  41. +184 −128 spec/unit/rpc/client_spec.rb
  42. +2 −2 spec/unit/ssl_spec.rb
  43. +6 −5 spec/unit/util_spec.rb
View
9 Rakefile
@@ -1,6 +1,11 @@
# Rakefile to build a project using HUDSON
-require 'rake/rdoctask'
+begin
+ require 'rdoc/task'
+rescue LoadError
+ require 'rake/rdoctask'
+end
+
require 'rake/packagetask'
require 'rake/clean'
require 'find'
@@ -73,7 +78,7 @@ task :default => [:clean, :doc, :package]
rd = Rake::RDocTask.new(:doc) { |rdoc|
rdoc.rdoc_dir = 'doc'
rdoc.title = "#{PROJ_DOC_TITLE} version #{CURRENT_VERSION}"
- rdoc.options << '--line-numbers' << '--inline-source' << '--main' << 'MCollective'
+ rdoc.options << '--line-numbers' << '--main' << 'MCollective'
RDOC_EXCLUDES.each do |ext|
rdoc.options << '--exclude' << ext
View
22 etc/discovery-help.erb
@@ -0,0 +1,22 @@
+<%= meta[:name] %>
+<% meta[:name].size.times do %>=<% end %>
+
+<%= meta[:description] %>
+
+ Author: <%= meta[:author] %>
+ Version: <%= meta[:version] %>
+ License: <%= meta[:license] %>
+ Timeout: <%= meta[:timeout] %>
+ Home Page: <%= meta[:url] %>
+
+DISCOVERY METHOD CAPABILITIES:
+% [["Filter based on configuration management classes", :classes],
+% ["Filter based on system facts", :facts],
+% ["Filter based on mcollective identity", :identity],
+% ["Filter based on mcollective agents", :agents],
+% ["Compound filters combining classes and facts", :compound]].each do |cap|
+% if entities[:discovery][:capabilities].include?(cap.last)
+ <%= cap.first %>
+% end
+% end
+
View
11 ext/debian/mcollective-common.install
@@ -1,10 +1,3 @@
usr/lib/ruby/1.8/* usr/lib/ruby/1.8/
-etc/mcollective/rpc-help.erb etc/mcollective
-etc/mcollective/data-help.erb etc/mcollective
-usr/share/mcollective/plugins/mcollective/agent usr/share/mcollective/plugins/mcollective
-usr/share/mcollective/plugins/mcollective/audit usr/share/mcollective/plugins/mcollective
-usr/share/mcollective/plugins/mcollective/connector usr/share/mcollective/plugins/mcollective
-usr/share/mcollective/plugins/mcollective/facts usr/share/mcollective/plugins/mcollective
-usr/share/mcollective/plugins/mcollective/registration usr/share/mcollective/plugins/mcollective
-usr/share/mcollective/plugins/mcollective/security usr/share/mcollective/plugins/mcollective
-usr/share/mcollective/plugins/mcollective/data usr/share/mcollective/plugins/mcollective
+etc/mcollective/*.erb etc/mcollective
+usr/share/mcollective/plugins/mcollective/* usr/share/mcollective/plugins/mcollective
View
10 ext/redhat/mcollective.spec
@@ -66,6 +66,7 @@ rm -rf %{buildroot}
%{__install} -m0444 etc/facts.yaml.dist %{buildroot}%{_sysconfdir}/mcollective/facts.yaml
%{__install} -m0444 etc/rpc-help.erb %{buildroot}%{_sysconfdir}/mcollective/rpc-help.erb
%{__install} -m0444 etc/data-help.erb %{buildroot}%{_sysconfdir}/mcollective/data-help.erb
+%{__install} -m0444 etc/discovery-help.erb %{buildroot}%{_sysconfdir}/mcollective/discovery-help.erb
%if 0%{?suse_version}
%{__install} -m0755 mcollective.init %{buildroot}%{_sysconfdir}/init.d/mcollective
%else
@@ -100,17 +101,12 @@ fi
%doc COPYING
%{ruby_sitelib}/mcollective.rb
%{ruby_sitelib}/mcollective
-%{_libexecdir}/mcollective/mcollective/agent
-%{_libexecdir}/mcollective/mcollective/audit
-%{_libexecdir}/mcollective/mcollective/connector
-%{_libexecdir}/mcollective/mcollective/facts
-%{_libexecdir}/mcollective/mcollective/registration
-%{_libexecdir}/mcollective/mcollective/security
-%{_libexecdir}/mcollective/mcollective/data
+%{_libexecdir}/mcollective/mcollective
%dir %{_sysconfdir}/mcollective
%dir %{_sysconfdir}/mcollective/ssl
%config%{_sysconfdir}/mcollective/rpc-help.erb
%config%{_sysconfdir}/mcollective/data-help.erb
+%config%{_sysconfdir}/mcollective/discovery-help.erb
%files client
%attr(0755, root, root)%{_sbindir}/mc-call-agent
View
22 ext/zsh/_mco
@@ -27,25 +27,33 @@ _mco() {
{-A,--wa,--with-agent}'[Agent filter]' \
{-I,--wi,--with-identity}'[Identity filter]' \
{-T,--target}'[Target collective]' \
+ {--dm,--disc-method}'[Which discovery method to use]' \
+ {--do,--disc-option}'[Options to pass to the discovery method]' \
{--dt,--discovery-timeout}'[Discovery timeout]' \
{-t,--timeout}'[Command Timeout]' \
{-q,--quiet}'[Surpress verbose output]' \
{-c,--config}'[Path to the config file]' \
{-v,--verbose}'[Be verbose]' \
{-h,--help}'[Show complete help message]' \
- '-ttl[Time To Live for the request]' \
+ '--nodes[List of nodes to address]' \
+ '--ttl[Time To Live for the request]' \
'--reply-to[Custom reply target]')
+ curcontext="${curcontext%:*:*}:mco-${application}"
+
if (( $+functions[_mco_application_$application] > 0 ));then
_mco_application_$application
fi
_arguments -s : $args
else
local -a cmdlist
- _call_program list-applications mco completion --list-applications -v | while read -A hline; do
+ _call_program mco-list-applications mco completion --list-applications -v | while read -A hline; do
cmdlist=($cmdlist "${hline}")
done
+
+ curcontext="${curcontext%:*:*}:mco-applications"
+
_describe -t mco-application 'MCollective applications' cmdlist
fi
}
@@ -54,24 +62,24 @@ _mco_application_rpc() {
local -a clist
if (( CURRENT == 3 )); then
- _call_program list-applications mco completion --list-agents -v | while read -A hline; do
+ _call_program mco-list-agents mco completion --list-agents -v | while read -A hline; do
clist=($clist "${hline}")
done
_describe -t mco-agents "MCollective agents" clist
elif (( CURRENT == 4 )); then
- _call_program list-applications mco completion --list-actions --agent=${words[2]} -v | while read -A hline; do
+ _call_program mco-list-actions mco completion --list-actions --agent=${words[2]} -v | while read -A hline; do
clist=($clist "${hline}")
done
- _describe -t mco-agents "${words[2]} actions" clist
+ _describe -t mco-actions "${words[2]} actions" clist
elif (( CURRENT > 4 )); then
- _call_program list-applications mco completion --list-inputs --action=${words[3]} --agent=${words[2]} -v | while read hline; do
+ _call_program mco-list-inputs mco completion --list-inputs --action=${words[3]} --agent=${words[2]} -v | while read hline; do
clist=($clist $hline)
done
- _describe -t mco-agents "${words[3]} inputs" clist -S =
+ _describe -t mco-inputs "${words[3]} inputs" clist -S =
fi
args+=(
View
1 lib/mcollective.rb
@@ -44,6 +44,7 @@ class UnknownRPCError<RPCError;end
autoload :Connector, "mcollective/connector"
autoload :Data, "mcollective/data"
autoload :DDL, "mcollective/ddl"
+ autoload :Discovery, "mcollective/discovery"
autoload :Facts, "mcollective/facts"
autoload :Logger, "mcollective/logger"
autoload :Log, "mcollective/log"
View
8 lib/mcollective/application.rb
@@ -255,7 +255,11 @@ def application_failure(e, err_dest=STDERR)
raise(e)
end
- err_dest.puts Util.colorize(:red, "#{$0} failed to run: #{e} ") + "(#{e.class.to_s})"
+ if options.nil? || options[:verbose]
+ err_dest.puts "\n%s %s" % [ Util.colorize(:red, e.to_s), Util.colorize(:bold, "(#{e.class.to_s})")]
+ else
+ err_dest.puts "\nThe %s application failed to run, use -v for full error details: %s\n" % [ Util.colorize(:bold, $0), Util.colorize(:red, e.to_s)]
+ end
if options.nil? || options[:verbose]
e.backtrace.each{|l| err_dest.puts "\tfrom #{l}"}
@@ -307,7 +311,7 @@ def main
# Exit with 0 if no discovery were done and > 0 responses were received
# Exit with 1 if no nodes were discovered
# Exit with 2 if nodes were discovered but some RPC requests failed
- # Exit with 3 if nodes were discovered, but not responses receivedif
+ # Exit with 3 if nodes were discovered, but not responses received
# Exit with 4 if no discovery were done and no responses were received
def halt(stats)
request_stats = {:discoverytime => 0,
View
28 lib/mcollective/client.rb
@@ -1,7 +1,7 @@
module MCollective
# Helpers for writing clients that can talk to agents, do discovery and so forth
class Client
- attr_accessor :options, :stats
+ attr_accessor :options, :stats, :discoverer
def initialize(configfile)
@config = Config.instance
@@ -14,6 +14,7 @@ def initialize(configfile)
@options = nil
@subscriptions = {}
+ @discoverer = Discovery.new(self)
@connection.connect
end
@@ -114,31 +115,10 @@ def receive(requestid = nil)
def discover(filter, timeout, limit=0)
raise "Limit has to be an integer" unless limit.is_a?(Fixnum)
- compount_timeout = timeout_for_compound_filter(options[:filter]["compound"])
+ compount_timeout = timeout_for_compound_filter(@options[:filter]["compound"])
timeout = timeout + compount_timeout
- begin
- hosts = []
- Timeout.timeout(timeout) do
- reqid = sendreq("ping", "discovery", filter)
- Log.debug("Waiting #{timeout} seconds for discovery replies to request #{reqid}")
-
- loop do
- reply = receive(reqid)
- Log.debug("Got discovery reply from #{reply.payload[:senderid]}")
- hosts << reply.payload[:senderid]
-
- return hosts if limit > 0 && hosts.size == limit
- end
- end
- rescue Timeout::Error => e
- rescue Exception => e
- raise
- ensure
- unsubscribe("discovery", :reply)
- end
-
- hosts.sort
+ discovered = @discoverer.discover(filter, timeout, limit)
end
# Send a request, performs the passed block for each response
View
10 lib/mcollective/config.rb
@@ -11,8 +11,8 @@ class Config
attr_reader :rpcauthorization, :color, :configfile, :rpchelptemplate
attr_reader :rpclimitmethod, :logger_type, :fact_cache_time, :collectives
attr_reader :main_collective, :ssl_cipher, :registration_collective
- attr_reader :direct_addressing, :direct_addressing_threshold
- attr_reader :ttl, :helptemplatedir, :queueprefix
+ attr_reader :direct_addressing, :direct_addressing_threshold, :ttl, :helptemplatedir
+ attr_reader :queueprefix, :default_discovery_method, :default_discovery_options
def initialize
@configured = false
@@ -109,6 +109,10 @@ def loadconfig(configfile)
@ttl = val.to_i
when "helptemplatedir"
@helptemplatedir = val
+ when "default_discovery_options"
+ @default_discovery_options << val
+ when "default_discovery_method"
+ @default_discovery_method = val
else
raise("Unknown config parameter #{key}")
end
@@ -174,6 +178,8 @@ def set_config_defaults(configfile)
@ssl_cipher = "aes-256-cbc"
@direct_addressing = false
@direct_addressing_threshold = 10
+ @default_discovery_method = "mc"
+ @default_discovery_options = []
@ttl = 60
# look in the config dir for the template so users can provide their own and windows
View
45 lib/mcollective/ddl.rb
@@ -99,10 +99,36 @@ def dataquery(input, &block)
@current_entity = nil
end
- # Returns the interface for the data query
- def dataquery_interface
- raise "Only data DDLs have data queries" unless @plugintype == :data
- @entities[:data] || {}
+ # Creates the definition for new discovery plugins
+ #
+ # discovery do
+ # capabilities [:classes, :facts, :identity, :agents, :compound]
+ # end
+ def discovery(&block)
+ raise "Discovery plugins can only have one definition" if @entities[:discovery]
+
+ @entities[:discovery] = {:capabilities => []}
+
+ @current_entity = :discovery
+ block.call if block_given?
+ @current_entity = nil
+ end
+
+ # records valid capabilities for discovery plugins
+ def capabilities(caps)
+ raise "Only discovery DDLs have capabilities" unless @plugintype == :discovery
+
+ caps = [caps].flatten
+
+ raise "Discovery plugin capabilities can't be empty" if caps.empty?
+
+ caps.each do |cap|
+ if [:classes, :facts, :identity, :agents, :compound].include?(cap)
+ @entities[:discovery][:capabilities] << cap
+ else
+ raise "%s is not a valid capability, valid capabilities are :classes, :facts, :identity, :agents and :compound" % cap
+ end
+ end
end
# Creates the definition for an action, you can nest input definitions inside the
@@ -240,12 +266,23 @@ def actions
@entities.keys
end
+ # Returns the interface for the data query
+ def dataquery_interface
+ raise "Only data DDLs have data queries" unless @plugintype == :data
+ @entities[:data] || {}
+ end
+
# Returns the interface for a specific action
def action_interface(name)
raise "Only agent DDLs have actions" unless @plugintype == :agent
@entities[name] || {}
end
+ def discovery_interface
+ raise "Only discovery DDLs have discovery interfaces" unless @plugintype == :discovery
+ @entities[:discovery]
+ end
+
# validate strings, lists and booleans, we'll add more types of validators when
# all the use cases are clear
#
View
116 lib/mcollective/discovery.rb
@@ -0,0 +1,116 @@
+module MCollective
+ class Discovery
+ def initialize(client)
+ @known_methods = find_known_methods
+ @default_method = Config.instance.default_discovery_method
+ @client = client
+ end
+
+ def find_known_methods
+ PluginManager.find("discovery")
+ end
+
+ def has_method?(method)
+ @known_methods.include?(method)
+ end
+
+ def force_direct_mode?
+ discovery_method != "mc"
+ end
+
+ def discovery_method
+ method = "mc"
+
+ if @client.options[:discovery_method]
+ method = @client.options[:discovery_method]
+ else
+ method = @default_method
+ end
+
+ raise "Unknown discovery method %s" % method unless has_method?(method)
+
+ unless method == "mc"
+ raise "Custom discovery methods require direct addressing mode" unless Config.instance.direct_addressing
+ end
+
+ return method
+ end
+
+ def discovery_class
+ method = discovery_method.capitalize
+
+ PluginManager.loadclass("MCollective::Discovery::#{method}") unless self.class.const_defined?(method)
+
+ self.class.const_get(method)
+ end
+
+ def ddl
+ @ddl ||= DDL.new(discovery_method, :discovery)
+
+ # if the discovery method got changed we might have an old DDL cached
+ # this will detect that and reread the correct DDL from disk
+ unless @ddl.meta[:name] == discovery_method
+ @ddl = DDL.new(discovery_method, :discovery)
+ end
+
+ return @ddl
+ end
+
+ # Agent filters are always present no matter what, so we cant raise an error if the capabilities
+ # suggest the discovery method cant do agents we just have to rely on the discovery plugin to not
+ # do stupid things in the presense of a agent filter
+ def check_capabilities(filter)
+ capabilities = ddl.discovery_interface[:capabilities]
+
+ unless capabilities.include?(:classes)
+ raise "Cannot use class filters while using the '%s' discovery method" % discovery_method unless filter["cf_class"].empty?
+ end
+
+ unless capabilities.include?(:facts)
+ raise "Cannot use fact filters while using the '%s' discovery method" % discovery_method unless filter["fact"].empty?
+ end
+
+ unless capabilities.include?(:identity)
+ raise "Cannot use identity filters while using the '%s' discovery method" % discovery_method unless filter["identity"].empty?
+ end
+
+ unless capabilities.include?(:compound)
+ raise "Cannot use compound filters while using the '%s' discovery method" % discovery_method unless filter["compound"].empty?
+ end
+ end
+
+ # checks if compound filters are used and then forces the 'mc' discovery plugin
+ def force_discovery_method_by_filter(filter)
+ unless discovery_method == "mc"
+ unless filter["compound"].empty?
+ Log.info "Switching to mc discovery method because compound filters are used"
+ @client.options[:discovery_method] = "mc"
+
+ return true
+ end
+ end
+
+ return false
+ end
+
+ def discover(filter, timeout, limit)
+ raise "Limit has to be an integer" unless limit.is_a?(Fixnum)
+
+ if force_discovery_method_by_filter(filter)
+ timeout = ddl.meta[:timeout] + @client.timeout_for_compound_filter(filter["compound"])
+ else
+ timeout = ddl.meta[:timeout] unless timeout
+ end
+
+ check_capabilities(filter)
+
+ discovered = discovery_class.discover(filter, timeout, limit, @client)
+
+ if limit > 0
+ return discovered[0,limit]
+ else
+ return discovered
+ end
+ end
+ end
+end
View
12 lib/mcollective/message.rb
@@ -72,15 +72,21 @@ def initialize(payload, message, options = {})
# this is to force a workflow that doesnt not yield in a mistake when someone might assume
# direct_addressing is enabled when its not.
def type=(type)
+ raise "Unknown message type #{type}" unless VALIDTYPES.include?(type)
+
if type == :direct_request
raise "Direct requests is not enabled using the direct_addressing config option" unless Config.instance.direct_addressing
unless @discovered_hosts && !@discovered_hosts.empty?
raise "Can only set type to :direct_request if discovered_hosts have been set"
end
- end
- raise "Unknown message type #{type}" unless VALIDTYPES.include?(type)
+ # clear out the filter, custom discovery sources might interpret the filters
+ # different than the remote mcollectived and in directed mode really the only
+ # filter that matters is the agent filter
+ @filter = Util.empty_filter
+ @filter["agent"] << @agent
+ end
@type = type
end
@@ -213,7 +219,7 @@ def publish
# send it as is.
if @discovered_hosts && Config.instance.direct_addressing
if @discovered_hosts.size <= Config.instance.direct_addressing_threshold
- @type = :direct_request
+ self.type = :direct_request
Log.debug("Handling #{requestid} as a direct request")
end
View
17 lib/mcollective/optionparser.rb
@@ -152,6 +152,23 @@ def add_common_options
@parser.on('--reply-to TARGET', 'Set a custom target for replies') do |v|
@options[:reply_to] = v
end
+
+ @parser.on('--dm', '--disc-method METHOD', 'Which discovery method to use') do |v|
+ raise "Discovery method is already set by a competing option" if @options[:discovery_method] && @options[:discovery_method] != v
+ @options[:discovery_method] = v
+ end
+
+ @parser.on('--do', '--disc-option OPTION', 'Options to pass to the discovery method') do |a|
+ @options[:discovery_options] << a
+ end
+
+ @parser.on("--nodes FILE", "List of nodes to address") do |v|
+ raise "Cannot mix --disc-method, --disc-option and --nodes" if @options[:discovery_method] || @options[:discovery_options].size > 0
+ raise "Cannot read the discovery file #{v}" unless File.readable?(v)
+
+ @options[:discovery_method] = "flatfile"
+ @options[:discovery_options] = v
+ end
end
private
View
84 lib/mcollective/rpc/client.rb
@@ -3,8 +3,9 @@ module RPC
# The main component of the Simple RPC client system, this wraps around MCollective::Client
# and just brings in a lot of convention and standard approached.
class Client
- attr_accessor :discovery_timeout, :timeout, :verbose, :filter, :config, :progress, :ttl, :reply_to
+ attr_accessor :timeout, :verbose, :filter, :config, :progress, :ttl, :reply_to
attr_reader :client, :stats, :ddl, :agent, :limit_targets, :limit_method, :output_format, :batch_size, :batch_sleep_time, :batch_mode
+ attr_reader :discovery_options, :discovery_method
@@initial_options = nil
@@ -37,9 +38,9 @@ def initialize(agent, flags = {})
@@initial_options = Marshal.dump(initial_options)
end
+ @initial_options = initial_options
@stats = Stats.new
@agent = agent
- @discovery_timeout = initial_options[:disctimeout] || 2
@timeout = initial_options[:timeout] || 5
@verbose = initial_options[:verbose]
@filter = initial_options[:filter]
@@ -51,6 +52,8 @@ def initialize(agent, flags = {})
@output_format = initial_options[:output_format] || :console
@force_direct_request = false
@reply_to = initial_options[:reply_to]
+ @discovery_method = initial_options[:discovery_method]
+ @discovery_options = initial_options[:discovery_options] || []
@batch_size = Integer(initial_options[:batch_size] || 0)
@batch_sleep_time = Float(initial_options[:batch_sleep_time] || 1)
@@ -61,6 +64,8 @@ def initialize(agent, flags = {})
@client = MCollective::Client.new(@config)
@client.options = initial_options
+ @discovery_timeout = discovery_timeout
+
@collective = @client.collective
@ttl = initial_options[:ttl] || Config.instance.ttl
@@ -319,6 +324,30 @@ def custom_request(action, args, expected_agents, filter = {}, &block)
end
end
+ def discovery_timeout
+ return @initial_options[:disctimeout] if @initial_options[:disctimeout]
+ return @client.discoverer.ddl.meta[:timeout]
+ end
+
+ def discovery_method=(method)
+ @discovery_method = method
+
+ if @initial_options[:discovery_options]
+ @discovery_options = @initial_options[:discovery_options]
+ else
+ @discovery_options.clear
+ end
+
+ @client.options = options
+ @discovery_timeout = discovery_timeout
+ reset
+ end
+
+ def discovery_options=(options)
+ @discovery_options = [options].flatten
+ reset
+ end
+
# Sets the class filter
def class_filter(klass)
@filter["cf_class"] << klass
@@ -446,8 +475,21 @@ def discover(flags={})
unless @discovered_agents
@stats.time_discovery :start
+ # if compound filters are used the only real option is to use the mc
+ # discovery plugin since its the only capable of using data queries etc
+ # and we do not want to degrade that experience just to allow compounds
+ # on other discovery plugins the UX would be too bad raising complex sets
+ # of errors etc.
+ @client.discoverer.force_discovery_method_by_filter(options[:filter])
+
actual_timeout = options[:disctimeout] + @client.timeout_for_compound_filter(options[:filter]["compound"])
- @stderr.print("Determining the amount of hosts matching filter for %d seconds .... " % actual_timeout) if verbose
+ if actual_timeout > 0
+ @stderr.print("Discovering hosts using the %s method for %d second(s) .... " % [@client.discoverer.discovery_method, actual_timeout]) if verbose
+ else
+ @stderr.print("Discovering hosts using the %s method .... " % [@client.discoverer.discovery_method]) if verbose
+ end
+
+ @client.options = options
# if the requested limit is a pure number and not a percent
# and if we're configured to use the first found hosts as the
@@ -459,9 +501,10 @@ def discover(flags={})
@discovered_agents = @client.discover(@filter, options[:disctimeout])
end
- @force_direct_request = false
@stderr.puts(@discovered_agents.size) if verbose
+ @force_direct_request = @client.discoverer.force_direct_mode?
+
@stats.time_discovery :end
end
@@ -481,6 +524,8 @@ def options
:collective => @collective,
:output_format => @output_format,
:ttl => @ttl,
+ :discovery_method => @discovery_method,
+ :discovery_options => @discovery_options,
:config => @config}
end
@@ -694,12 +739,13 @@ def call_agent(action, args, opts, disc=:auto, &block)
message = Message.new(req, nil, {:agent => @agent, :type => :request, :collective => @collective, :filter => opts[:filter], :options => opts})
message.discovered_hosts = discovered.clone
- message.type = :direct_request if @force_direct_request
result = []
respcount = 0
if discovered.size > 0
+ message.type = :direct_request if @force_direct_request
+
if @progress && !block_given?
twirl = Progress.new
@stdout.puts
@@ -767,25 +813,25 @@ def process_results_with_block(action, resp, block)
@stats.time_block_execution :start
case block.arity
- when 1
- block.call(resp)
- when 2
- rpcresp = Result.new(@agent, action, {:sender => resp[:senderid], :statuscode => resp[:body][:statuscode],
- :statusmsg => resp[:body][:statusmsg], :data => resp[:body][:data]})
- block.call(resp, rpcresp)
+ when 1
+ block.call(resp)
+ when 2
+ rpcresp = Result.new(@agent, action, {:sender => resp[:senderid], :statuscode => resp[:body][:statuscode],
+ :statusmsg => resp[:body][:statusmsg], :data => resp[:body][:data]})
+ block.call(resp, rpcresp)
end
@stats.time_block_execution :end
else
case resp[:body][:statuscode]
- when 2
- raise UnknownRPCAction, resp[:body][:statusmsg]
- when 3
- raise MissingRPCData, resp[:body][:statusmsg]
- when 4
- raise InvalidRPCData, resp[:body][:statusmsg]
- when 5
- raise UnknownRPCError, resp[:body][:statusmsg]
+ when 2
+ raise UnknownRPCAction, resp[:body][:statusmsg]
+ when 3
+ raise MissingRPCData, resp[:body][:statusmsg]
+ when 4
+ raise InvalidRPCData, resp[:body][:statusmsg]
+ when 5
+ raise UnknownRPCError, resp[:body][:statusmsg]
end
end
end
View
4 lib/mcollective/rpc/progress.rb
@@ -40,9 +40,9 @@ def twirl(current, total)
return "\r#{current} / #{total}" if @size == 0
if current == total
- txt = "\r " + Util.colorize(:green, "*") + " [ "
+ txt = "\r %s [" % Util.colorize(:green, "*")
else
- txt = "\r #{@twirl[@twirldex]} [ "
+ txt = "\r %s [" % Util.colorize(:red, @twirl[@twirldex])
end
dashes = ((current.to_f / total) * @size).round
View
6 lib/mcollective/runner.rb
@@ -50,10 +50,10 @@ def run
begin
request = receive
- if request.agent == "mcollective"
- controlmsg(request)
- else
+ unless request.agent == "mcollective"
agentmsg(request)
+ else
+ Log.error("Received a control message, possibly via 'mco controller' but this has been deprecated")
end
rescue SignalException => e
Log.warn("Exiting after signal: #{e}")
View
52 lib/mcollective/security/base.rb
@@ -80,20 +80,20 @@ def validate_filter?(filter)
begin
compound.each do |expression|
case expression.keys.first
- when "statement"
- truth_values << Matcher.eval_compound_statement(expression).to_s
- when "fstatement"
- truth_values << Matcher.eval_compound_fstatement(expression.values.first)
- when "and"
- truth_values << "&&"
- when "or"
- truth_values << "||"
- when "("
- truth_values << "("
- when ")"
- truth_values << ")"
- when "not"
- truth_values << "!"
+ when "statement"
+ truth_values << Matcher.eval_compound_statement(expression).to_s
+ when "fstatement"
+ truth_values << Matcher.eval_compound_fstatement(expression.values.first)
+ when "and"
+ truth_values << "&&"
+ when "or"
+ truth_values << "||"
+ when "("
+ truth_values << "("
+ when ")"
+ truth_values << ")"
+ when "not"
+ truth_values << "!"
end
end
@@ -168,24 +168,24 @@ def create_reply(reqid, agent, body)
Log.debug("Encoded a message for request #{reqid}")
{:senderid => @config.identity,
- :requestid => reqid,
- :senderagent => agent,
- :msgtime => Time.now.utc.to_i,
- :body => body}
+ :requestid => reqid,
+ :senderagent => agent,
+ :msgtime => Time.now.utc.to_i,
+ :body => body}
end
def create_request(reqid, filter, msg, initiated_by, target_agent, target_collective, ttl=60)
Log.debug("Encoding a request for agent '#{target_agent}' in collective #{target_collective} with request id #{reqid}")
{:body => msg,
- :senderid => @config.identity,
- :requestid => reqid,
- :filter => filter,
- :collective => target_collective,
- :agent => target_agent,
- :callerid => callerid,
- :ttl => ttl,
- :msgtime => Time.now.utc.to_i}
+ :senderid => @config.identity,
+ :requestid => reqid,
+ :filter => filter,
+ :collective => target_collective,
+ :agent => target_agent,
+ :callerid => callerid,
+ :ttl => ttl,
+ :msgtime => Time.now.utc.to_i}
end
# Give a MC::Message instance and a message id this will figure out if you the incoming
View
24 lib/mcollective/util.rb
@@ -129,10 +129,10 @@ def self.empty_filter?(filter)
# Creates an empty filter
def self.empty_filter
{"fact" => [],
- "cf_class" => [],
- "agent" => [],
- "identity" => [],
- "compound" => []}
+ "cf_class" => [],
+ "agent" => [],
+ "identity" => [],
+ "compound" => []}
end
# Picks a config file defaults to ~/.mcollective
@@ -156,12 +156,14 @@ def self.config_file_for_user
# Creates a standard options hash
def self.default_options
- {:verbose => false,
- :disctimeout => 2,
- :timeout => 5,
- :config => config_file_for_user,
- :collective => nil,
- :filter => empty_filter}
+ {:verbose => false,
+ :disctimeout => nil,
+ :timeout => 5,
+ :config => config_file_for_user,
+ :collective => nil,
+ :discovery_method => nil,
+ :discovery_options => Config.instance.default_discovery_options,
+ :filter => empty_filter}
end
def self.make_subscriptions(agent, type, collective=nil)
@@ -268,7 +270,7 @@ def self.color(code)
# Helper to return a string in specific color
def self.colorize(code, msg)
- "%s%s%s" % [ self.color(code), msg, self.color(:reset) ]
+ "%s%s%s" % [ color(code), msg, color(:reset) ]
end
# Returns the current ruby version as per RUBY_VERSION, mostly
View
92 plugins/mcollective/application/controller.rb
@@ -1,92 +0,0 @@
-require 'pp'
-
-module MCollective
- class Application::Controller < Application
- description "Control the mcollective daemon"
-
- usage <<-END_OF_USAGE
-mco controller [OPTIONS] [FILTERS] <COMMAND> [--argument <ARGUMENT>]
-
-The COMMAND can be one of the following:
-
- stats - retrieve statistics from the mcollectived
- reload_agent - reloads an agent, requires an agent name as argument
- reload_agents - reloads all agents
- END_OF_USAGE
-
- option :argument,
- :description => "Argument to pass to an agent",
- :arguments => [ '-a', '--arg', '--argument ARGUMENT' ],
- :type => String
-
- def print_statistics(sender, statistics)
- printf("%40s> total=%d, replies=%d, valid=%d, invalid=%d, " +
- "filtered=%d, passed=%d\n", sender,
- statistics[:total], statistics[:replies],
- statistics[:validated], statistics[:unvalidated],
- statistics[:filtered], statistics[:passed])
- end
-
- def post_option_parser(configuration)
- configuration[:command] = ARGV.shift if ARGV.size > 0
- end
-
- def validate_configuration(configuration)
- unless configuration.include?(:command)
- raise "Please specify a command and optional arguments"
- end
-
- #
- # When asked to restart an agent we need to make sure that
- # we have this agent name and set appropriate filters ...
- #
- if configuration[:command].match(/^reload_agent$/)
- unless configuration.include?(:argument)
- raise "Please specify an agent name to reload with --argument"
- end
-
- options[:filter]['agent'] << configuration[:argument]
- end
- end
-
- def main
- client = MCollective::Client.new(options[:config])
- client.options = options
-
- counter = 0
-
- command = configuration[:command]
- command += " #{configuration[:argument]}" if configuration[:argument]
-
- statistics = client.discovered_req(command, 'mcollective') do |response|
- next unless response
-
- counter += 1
-
- sender = response[:senderid]
- body = response[:body]
-
- case command
- when /^stats$/
- print_statistics(sender, body[:stats])
- when /^reload_agent(?:.+)/
- printf("%40s> %s\n", sender, body)
- else
- if options[:verbose]
- puts "#{sender}>"
- pp body
- else
- puts if counter % 4 == 1
- print "#{sender} "
- end
- end
- end
-
- client.disconnect
-
- client.display_stats(statistics, false, "mcollectived controller summary")
-
- halt statistics
- end
- end
-end
View
21 plugins/mcollective/application/find.rb
@@ -1,16 +1,21 @@
class MCollective::Application::Find<MCollective::Application
- description "Find hosts matching criteria"
+ description "Find hosts using the discovery system matching filter criteria"
def main
- client = MCollective::Client.new(options[:config])
- client.options = options
+ mc = rpcclient("rpcutil")
- stats = client.req("ping", "discovery") do |resp|
- puts resp[:senderid]
- end
+ starttime = Time.now
- client.display_stats(stats) if options[:verbose]
+ nodes = mc.discover
- halt stats
+ discoverytime = Time.now - starttime
+
+ STDERR.puts if options[:verbose]
+
+ nodes.each {|c| puts c}
+
+ STDERR.puts "\nDiscovered %s nodes in %.2f seconds using the %s discovery plugin" % [nodes.size, discoverytime, mc.client.discoverer.discovery_method] if options[:verbose]
+
+ nodes.size > 0 ? exit(0) : exit(1)
end
end
View
4 plugins/mcollective/application/plugin.rb
@@ -111,7 +111,7 @@ def package_command
# Show application list and plugin help
def doc_command
- known_plugin_types = [["Agents", :agent], ["Data Queries", :data]]
+ known_plugin_types = [["Agents", :agent], ["Data Queries", :data], ["Discovery Methods", :discovery]]
if configuration.include?(:target) && configuration[:target] != "."
if configuration[:target] =~ /^(.+?)\/(.+)$/
@@ -205,7 +205,7 @@ def set_plugin_type
# To keep it simple we limit it to one type per target directory.
def identify_plugin
plugintype = Dir.glob(File.join(configuration[:target], "*")).select do |file|
- File.directory?(file) && file.match(/(connector|facts|registration|security|audit|pluginpackager|data)/)
+ File.directory?(file) && file.match(/(connector|facts|registration|security|audit|pluginpackager|data|discovery)/)
end
raise RuntimeError, "more than one plugin type detected in directory" if plugintype.size > 1
View
11 plugins/mcollective/discovery/flatfile.ddl
@@ -0,0 +1,11 @@
+metadata :name => "flatfile",
+ :description => "Flatfile based discovery for node identities",
+ :author => "R.I.Pienaar <rip@devco.net>",
+ :license => "ASL 2.0",
+ :version => "0.1",
+ :url => "http://marionette-collective.org/",
+ :timeout => 0
+
+discovery do
+ capabilities :identity
+end
View
40 plugins/mcollective/discovery/flatfile.rb
@@ -0,0 +1,40 @@
+# discovers against a flatfile instead of the traditional network discovery
+# the flat file must have a node name per line which should match identities
+# as configured
+module MCollective
+ class Discovery
+ class Flatfile
+ def self.discover(filter, timeout, limit=0, client=nil)
+ unless client.options[:discovery_options].empty?
+ file = client.options[:discovery_options].first
+ else
+ raise "The flatfile discovery method needs a path to a text file"
+ end
+
+ raise "Cannot read the file %s specified as discovery source" % file unless File.readable?(file)
+
+ discovered = []
+
+ hosts = File.readlines(file).map{|l| l.chomp}
+
+ # this plugin only supports identity filters, do regex matches etc against
+ # the list found in the flatfile
+ unless filter["identity"].empty?
+ filter["identity"].each do |identity|
+ identity = Regexp.new(identity.gsub("\/", "")) if identity.match("^/")
+
+ if identity.is_a?(Regexp)
+ discovered = hosts.grep(identity)
+ elsif hosts.include?(identity)
+ discovered << identity
+ end
+ end
+ else
+ discovered = hosts
+ end
+
+ discovered
+ end
+ end
+ end
+end
View
11 plugins/mcollective/discovery/mc.ddl
@@ -0,0 +1,11 @@
+metadata :name => "mc",
+ :description => "MCollective Broadcast based discovery",
+ :author => "R.I.Pienaar <rip@devco.net>",
+ :license => "ASL 2.0",
+ :version => "0.1",
+ :url => "http://marionette-collective.org/",
+ :timeout => 2
+
+discovery do
+ capabilities [:classes, :facts, :identity, :agents, :compound]
+end
View
30 plugins/mcollective/discovery/mc.rb
@@ -0,0 +1,30 @@
+module MCollective
+ class Discovery
+ class Mc
+ def self.discover(filter, timeout, limit, client)
+ begin
+ hosts = []
+ Timeout.timeout(timeout) do
+ reqid = client.sendreq("ping", "discovery", filter)
+ Log.debug("Waiting #{timeout} seconds for discovery replies to request #{reqid}")
+
+ loop do
+ reply = client.receive(reqid)
+ Log.debug("Got discovery reply from #{reply.payload[:senderid]}")
+ hosts << reply.payload[:senderid]
+
+ return hosts if limit > 0 && hosts.size == limit
+ end
+ end
+ rescue Timeout::Error => e
+ rescue Exception => e
+ raise
+ ensure
+ client.unsubscribe("discovery", :reply)
+ end
+
+ hosts
+ end
+ end
+ end
+end
View
1 spec/spec_helper.rb
@@ -21,6 +21,7 @@
config.mock_with :mocha
config.before :each do
+ MCollective::Config.instance.set_config_defaults("")
MCollective::PluginManager.clear
end
end
View
36 spec/unit/agents_spec.rb
@@ -27,15 +27,15 @@ module MCollective
describe "#initialize" do
it "should fail if configuration has not been loaded" do
- Config.any_instance.expects(:configured).returns(false)
+ Config.instance.expects(:configured).returns(false)
expect {
Agents.new
}.to raise_error("Configuration has not been loaded, can't load agents")
end
it "should load agents" do
- Config.any_instance.expects(:configured).returns(true)
+ Config.instance.expects(:configured).returns(true)
Agents.any_instance.expects(:loadagents).once
Agents.new
@@ -44,8 +44,8 @@ module MCollective
describe "#clear!" do
it "should delete and unsubscribe all loaded agents" do
- Config.any_instance.expects(:configured).returns(true).at_least_once
- Config.any_instance.expects(:libdir).returns([@tmpdir])
+ Config.instance.expects(:configured).returns(true).at_least_once
+ Config.instance.expects(:libdir).returns([@tmpdir])
PluginManager.expects(:delete).with("foo_agent").once
Util.expects(:make_subscriptions).with("foo", :broadcast).returns("foo_target")
Util.expects(:unsubscribe).with("foo_target")
@@ -56,8 +56,8 @@ module MCollective
describe "#loadagents" do
before do
- Config.any_instance.stubs(:configured).returns(true)
- Config.any_instance.stubs(:libdir).returns([@tmpdir])
+ Config.instance.stubs(:configured).returns(true)
+ Config.instance.stubs(:libdir).returns([@tmpdir])
Agents.any_instance.stubs("clear!").returns(true)
end
@@ -67,7 +67,7 @@ module MCollective
end
it "should attempt to load agents from all libdirs" do
- Config.any_instance.expects(:libdir).returns(["/nonexisting", "/nonexisting"])
+ Config.instance.expects(:libdir).returns(["/nonexisting", "/nonexisting"])
File.expects("directory?").with("/nonexisting/mcollective/agent").twice
a = Agents.new
@@ -94,8 +94,8 @@ module MCollective
describe "#loadagent" do
before do
FileUtils.touch(File.join([@agentsdir, "test.rb"]))
- Config.any_instance.stubs(:configured).returns(true)
- Config.any_instance.stubs(:libdir).returns([@tmpdir])
+ Config.instance.stubs(:configured).returns(true)
+ Config.instance.stubs(:libdir).returns([@tmpdir])
Agents.any_instance.stubs("clear!").returns(true)
PluginManager.stubs(:loadclass).returns(true)
Util.stubs(:make_subscriptions).with("test", :broadcast).returns([{:agent => "test", :type => :broadcast, :collective => "test"}])
@@ -183,15 +183,15 @@ module MCollective
describe "#class_for_agent" do
it "should return the correct class" do
- Config.any_instance.stubs(:configured).returns(true)
+ Config.instance.stubs(:configured).returns(true)
Agents.any_instance.stubs(:loadagents).returns(true)
Agents.new.class_for_agent("foo").should == "MCollective::Agent::Foo"
end
end
describe "#activate_agent?" do
before do
- Config.any_instance.stubs(:configured).returns(true)
+ Config.instance.stubs(:configured).returns(true)
Agents.any_instance.stubs(:loadagents).returns(true)
@a = Agents.new
@@ -225,14 +225,14 @@ class MCollective::Agent::Test; end
describe "#findagentfile" do
before do
- Config.any_instance.stubs(:configured).returns(true)
- Config.any_instance.stubs(:libdir).returns([@tmpdir])
+ Config.instance.stubs(:configured).returns(true)
+ Config.instance.stubs(:libdir).returns([@tmpdir])
Agents.any_instance.stubs(:loadagents).returns(true)
@a = Agents.new
end
it "should support multiple libdirs" do
- Config.any_instance.expects(:libdir).returns([@tmpdir, @tmpdir]).once
+ Config.instance.expects(:libdir).returns([@tmpdir, @tmpdir]).once
File.expects("exist?").returns(false).twice
@a.findagentfile("test")
end
@@ -255,8 +255,8 @@ class MCollective::Agent::Test; end
describe "#include?" do
it "should correctly report the plugin state" do
- Config.any_instance.stubs(:configured).returns(true)
- Config.any_instance.stubs(:libdir).returns([@tmpdir])
+ Config.instance.stubs(:configured).returns(true)
+ Config.instance.stubs(:libdir).returns([@tmpdir])
Agents.any_instance.stubs(:loadagents).returns(true)
PluginManager.expects("include?").with("test_agent").returns(true)
@@ -268,8 +268,8 @@ class MCollective::Agent::Test; end
describe "#agentlist" do
it "should return the correct agent list" do
- Config.any_instance.stubs(:configured).returns(true)
- Config.any_instance.stubs(:libdir).returns([@tmpdir])
+ Config.instance.stubs(:configured).returns(true)
+ Config.instance.stubs(:libdir).returns([@tmpdir])
Agents.any_instance.stubs(:loadagents).returns(true)
@a = Agents.new("test" => true)
View
6 spec/unit/application_spec.rb
@@ -550,13 +550,14 @@ module MCollective
out = StringIO.new
@app.stubs(:disconnect)
@app.stubs(:exit).with(1)
+ @app.stubs(:options).returns({})
Config.instance.stubs(:color).returns(false)
e = mock
e.stubs(:backtrace).returns([])
e.stubs(:to_s).returns("rspec")
- out.expects(:puts).with("#{$0} failed to run: rspec (Mocha::Mock)")
+ out.expects(:puts).with(regexp_matches(/rspec application failed to run/))
@app.application_failure(e, out)
end
@@ -565,14 +566,15 @@ module MCollective
out = StringIO.new
@app.stubs(:disconnect)
@app.stubs(:exit).with(1)
+ @app.stubs(:options).returns(nil)
Config.instance.stubs(:color).returns(false)
e = mock
e.stubs(:backtrace).returns(["rspec"])
e.stubs(:to_s).returns("rspec")
@app.expects(:options).returns({:verbose => true}).twice
- out.expects(:puts).with("#{$0} failed to run: rspec (Mocha::Mock)")
+ out.expects(:puts).with(regexp_matches(/rspec.+Mocha::Mock/))
out.expects(:puts).with("\tfrom rspec")
@app.application_failure(e, out)
View
4 spec/unit/applications_spec.rb
@@ -111,13 +111,13 @@ module MCollective
describe "#list" do
it "should load the configuration" do
Applications.expects("load_config").returns(true).once
- Config.any_instance.expects("libdir").returns([@tmpdir])
+ Config.instance.expects("libdir").returns([@tmpdir])
Applications.list
end
it "should add found applications to the list" do
Applications.expects("load_config").returns(true).once
- Config.any_instance.expects("libdir").returns([@tmpdir])
+ Config.instance.expects("libdir").returns([@tmpdir])
Applications.list.should == ["test"]
end
View
32 spec/unit/client_spec.rb
@@ -4,6 +4,37 @@
module MCollective
describe Client do
+ describe "#discover" do
+ before do
+ @security = mock
+ @security.expects(:initiated_by=)
+ @connector = mock
+ @connector.expects(:connect)
+ @ddl = mock
+ @ddl.stubs(:meta).returns({:timeout => 1})
+ @discoverer = mock
+
+ Discovery.expects(:new).returns(@discoverer)
+
+ Config.instance.instance_variable_set("@configured", true)
+ PluginManager.expects("[]").with("connector_plugin").returns(@connector)
+ PluginManager.expects("[]").with("security_plugin").returns(@security)
+
+ @client = Client.new("/nonexisting")
+ end
+
+ it "should not allow non integer limits" do
+ expect { @client.discover(nil, nil, 1.1) }.to raise_error("Limit has to be an integer")
+ end
+
+ it "should calculate the correct timeout" do
+ @client.expects(:timeout_for_compound_filter).returns(1)
+ @client.options = {:filter => {"compound" => {}}}
+ @discoverer.expects(:discover).with({}, 2, 0).returns([])
+ @client.discover({}, 1)
+ end
+ end
+
describe "#timeout_for_compound_filter" do
it "should return the correct time" do
security = mock
@@ -12,6 +43,7 @@ module MCollective
connector.expects(:connect)
ddl = mock
ddl.stubs(:meta).returns({:timeout => 1})
+ Discovery.expects(:new).returns(nil)
Config.instance.instance_variable_set("@configured", true)
PluginManager.expects("[]").with("connector_plugin").returns(connector)
View
11 spec/unit/config_spec.rb
@@ -75,6 +75,17 @@ module MCollective
Config.instance.loadconfig("/nonexisting")
Config.instance.rpchelptemplate.should == "/etc/mcollective/rpc-help.erb"
end
+
+ it "should support multiple default_discovery_options" do
+ File.expects(:open).with("/nonexisting", "r").returns(StringIO.new("default_discovery_options = 1\ndefault_discovery_options = 2"))
+ File.expects(:exists?).with("/nonexisting").returns(true)
+ File.expects(:exists?).with(File.join(File.dirname("/nonexisting"), "rpc-help.erb")).returns(true)
+ PluginManager.stubs(:loadclass)
+ PluginManager.stubs("<<")
+
+ Config.instance.loadconfig("/nonexisting")
+ Config.instance.default_discovery_options.should == ["1", "2"]
+ end
end
describe "#read_plugin_config_dir" do
View
8 spec/unit/data/base_spec.rb
@@ -10,14 +10,6 @@ module Data
@ddl.stubs(:meta).returns({:timeout => 1})
end
- describe "#inherited" do
- it "should add classes to the plugin manager" do
- PluginManager.expects("<<").with(:type => "rspec_data", :class => "MCollective::Data::Rspec_data", :single_instance => false)
-
- class Rspec_data<Base; end
- end
- end
-
describe "#initialize" do
it "should set the plugin name, ddl and timeout and call the startup hook" do
DDL.stubs(:new).returns(@ddl)
View
58 spec/unit/ddl_spec.rb
@@ -11,22 +11,22 @@ module MCollective
describe "#findddlfile" do
it "should construct the correct ddl file name" do
- Config.any_instance.expects(:libdir).returns(["/nonexisting"])
+ Config.instance.expects(:libdir).returns(["/nonexisting"])
File.expects("exist?").with("/nonexisting/mcollective/agent/foo.ddl").returns(false)
@ddl.findddlfile("foo").should == false
end
it "should check each libdir for a ddl file" do
- Config.any_instance.expects(:libdir).returns(["/nonexisting1", "/nonexisting2"])
+ Config.instance.expects(:libdir).returns(["/nonexisting1", "/nonexisting2"])
File.expects("exist?").with("/nonexisting1/mcollective/agent/foo.ddl").returns(false)
File.expects("exist?").with("/nonexisting2/mcollective/agent/foo.ddl").returns(false)
@ddl.findddlfile("foo").should == false
end
it "should return the ddl file path if found" do
- Config.any_instance.expects(:libdir).returns(["/nonexisting"])
+ Config.instance.expects(:libdir).returns(["/nonexisting"])
File.expects("exist?").with("/nonexisting/mcollective/agent/foo.ddl").returns(true)
Log.expects(:debug).with("Found foo ddl at /nonexisting/mcollective/agent/foo.ddl")
@@ -104,6 +104,57 @@ module MCollective
end
end
+ describe "#discovery_interface" do
+ it "should fail for non discovery plugin DDLs" do
+ expect { @ddl.discovery_interface }.to raise_error("Only discovery DDLs have discovery interfaces")
+ end
+
+ it "should return correct data" do
+ @ddl.instance_variable_set("@plugintype", :discovery)
+ @ddl.discovery do
+ @ddl.capabilities :identity
+ end
+
+ @ddl.discovery_interface.should == {:capabilities => [:identity]}
+ end
+ end
+
+ describe "#capabilities" do
+ it "should fail on non discovery plugins" do
+ expect { @ddl.capabilities :rspec }.to raise_error("Only discovery DDLs have capabilities")
+ end
+
+ it "should support non arrays" do
+ @ddl.instance_variable_set("@plugintype", :discovery)
+ @ddl.discovery do
+ @ddl.capabilities :identity
+ end
+ @ddl.discovery_interface.should == {:capabilities => [:identity]}
+ end
+
+ it "should not accept empty capability lists" do
+ @ddl.instance_variable_set("@plugintype", :discovery)
+ @ddl.discovery do
+ expect { @ddl.capabilities [] }.to raise_error("Discovery plugin capabilities can't be empty")
+ end
+ end
+
+ it "should only accept known capabilities" do
+ @ddl.instance_variable_set("@plugintype", :discovery)
+ @ddl.discovery do
+ expect { @ddl.capabilities :rspec }.to raise_error(/rspec is not a valid capability/)
+ end
+ end
+
+ it "should correctly store the capabilities" do
+ @ddl.instance_variable_set("@plugintype", :discovery)
+ @ddl.discovery do
+ @ddl.capabilities [:identity, :classes]
+ end
+ @ddl.discovery_interface.should == {:capabilities => [:identity, :classes]}
+ end
+ end
+
describe "#dataquery_interface" do
it "should fail for non data plugins" do
expect { @ddl.dataquery_interface }.to raise_error("Only data DDLs have data queries")
@@ -277,7 +328,6 @@ module MCollective
describe "#help" do
it "should use conventional template paths when none is provided" do
- Config.instance.set_config_defaults("")
File.expects(:read).with("/etc/mcollective/rpc-help.erb").returns("rspec")
@ddl.help.should == "rspec"
end
View
191 spec/unit/discovery_spec.rb
@@ -0,0 +1,191 @@
+#!/usr/bin/env rspec
+
+require 'spec_helper'
+
+module MCollective
+ describe Discovery do
+ before do
+ Config.instance.stubs(:default_discovery_method).returns("mc")
+ @client = mock
+
+ Discovery.any_instance.stubs(:find_known_methods).returns(["mc"])
+ @discovery = Discovery.new(@client)
+ end
+
+ describe "#discover" do
+ before do
+ ddl = mock
+ ddl.stubs(:meta).returns({:timeout => 2})
+
+ discoverer = mock
+
+ @discovery.stubs(:force_discovery_method_by_filter).returns(false)
+ @discovery.stubs(:ddl).returns(ddl)
+ @discovery.stubs(:check_capabilities)
+ @discovery.stubs(:discovery_class).returns(discoverer)
+ end
+
+ it "should error for non fixnum limits" do
+ expect { @discovery.discover(nil, 0, 1.1) }.to raise_error("Limit has to be an integer")
+ end
+
+ it "should calculate the correct timeout when forcing the method to mc" do
+ @discovery.expects(:force_discovery_method_by_filter).returns(true)
+ @client.expects(:timeout_for_compound_filter).returns(1)
+
+ filter = Util.empty_filter.merge({"compound" => "rspec"})
+ @discovery.discovery_class.expects(:discover).with(filter, 3, 0, @client)
+ @discovery.discover(filter, 1, 0)
+ end
+
+ it "should use the DDL timeout if none is specified" do
+ filter = Util.empty_filter
+ @discovery.discovery_class.expects(:discover).with(filter, 2, 0, @client)
+ @discovery.discover(filter, nil, 0)
+ end
+
+ it "should check the discovery method is capable of serving the filter" do
+ @discovery.expects(:check_capabilities).with("filter").raises("capabilities check failed")
+ expect { @discovery.discover("filter", nil, 0) }.to raise_error("capabilities check failed")
+ end
+
+ it "should call the correct discovery plugin" do
+ @discovery.discovery_class.expects(:discover).with("filter", 2, 0, @client)
+ @discovery.discover("filter", nil, 0)
+ end
+
+ it "should handle limits correctly" do
+ @discovery.discovery_class.stubs(:discover).returns([1,2,3,4,5])
+ @discovery.discover(Util.empty_filter, 1, 1).should == [1]
+ @discovery.discover(Util.empty_filter, 1, 0).should == [1,2,3,4,5]
+ end
+ end
+
+ describe "#force_discovery_method_by_filter" do
+ it "should force mc plugin when needed" do
+ options = {:discovery_method => "rspec"}
+
+ Log.expects(:info).with("Switching to mc discovery method because compound filters are used")
+
+ @discovery.expects(:discovery_method).returns("rspec")
+ @client.expects(:options).returns(options)
+ @discovery.force_discovery_method_by_filter({"compound" => ["rspec"]}).should == true
+
+ options[:discovery_method].should == "mc"
+ end
+
+ it "should not force mc plugin when no compound filter is used" do
+ options = {:discovery_method => "rspec"}
+
+ @discovery.expects(:discovery_method).returns("rspec")
+ @discovery.force_discovery_method_by_filter({"compound" => []}).should == false
+
+ options[:discovery_method].should == "rspec"
+ end
+ end
+
+ describe "#check_capabilities" do
+ before do
+ @ddl = mock
+ @discovery.stubs(:ddl).returns(@ddl)
+ @discovery.stubs(:discovery_method).returns("rspec")
+ end
+
+ it "should fail for unsupported capabilities" do
+ @ddl.stubs(:discovery_interface).returns({:capabilities => []})
+
+ filter = Util.empty_filter
+
+ expect { @discovery.check_capabilities(filter.merge({"cf_class" => ["filter"]})) }.to raise_error(/Cannot use class filters/)
+
+ ["fact", "identity", "compound"].each do |type|
+ expect { @discovery.check_capabilities(filter.merge({type => ["filter"]})) }.to raise_error(/Cannot use #{type} filters/)
+ end
+ end
+ end
+
+ describe "#ddl" do
+ before do
+ @ddl = mock
+ @ddl.stubs(:meta).returns({:name => "mc"})
+ end
+
+ it "should create an instance of the right ddl" do
+ @discovery.instance_variable_set("@ddl", nil)
+ @client.stubs(:options).returns({})
+ DDL.expects(:new).with("mc", :discovery).returns(@ddl)
+ @discovery.ddl
+ end
+
+ it "should reload the ddl if the method has changed" do
+ @discovery.instance_variable_set("@ddl", @ddl)
+ @discovery.stubs(:discovery_method).returns("rspec")
+ DDL.expects(:new).with("rspec", :discovery).returns(@ddl)
+ @discovery.ddl
+ end
+ end
+
+ describe "#discovery_class" do
+ it "should try to load the class if not already loaded" do
+ @discovery.expects(:discovery_method).returns("mc")
+ PluginManager.expects(:loadclass).with("MCollective::Discovery::Mc")
+ Discovery.expects(:const_defined?).with("Mc").returns(false)
+ Discovery.expects(:const_get).with("Mc").returns("rspec")
+ @discovery.discovery_class.should == "rspec"
+ end
+
+ it "should not load the class again if its already loaded" do
+ @discovery.expects(:discovery_method).returns("mc")
+ PluginManager.expects(:loadclass).never
+ Discovery.expects(:const_defined?).with("Mc").returns(true)
+ Discovery.expects(:const_get).with("Mc").returns("rspec")
+ @discovery.discovery_class.should == "rspec"
+ end
+ end
+
+ describe "#initialize" do
+ it "should load all the known methods" do
+ @discovery.instance_variable_get("@known_methods").should == ["mc"]
+ end
+ end
+
+ describe "#find_known_methods" do
+ it "should use the PluginManager to find plugins of type 'discovery'" do
+ @discovery.find_known_methods.should == ["mc"]
+ end
+ end
+
+ describe "#has_method?" do
+ it "should correctly report the availability of a discovery method" do
+ @discovery.has_method?("mc").should == true
+ @discovery.has_method?("rspec").should == false
+ end
+ end
+
+ describe "#descovery_method" do
+ it "should default to 'mc'" do
+ @client.expects(:options).returns({})
+ @discovery.discovery_method.should == "mc"
+ end
+
+ it "should give preference to the client options" do
+ @client.expects(:options).returns({:discovery_method => "rspec"}).twice
+ Config.instance.expects(:direct_addressing).returns(true)
+ @discovery.expects(:has_method?).with("rspec").returns(true)
+ @discovery.discovery_method.should == "rspec"
+ end
+
+ it "should validate the discovery method exists" do
+ @client.expects(:options).returns({:discovery_method => "rspec"}).twice
+ expect { @discovery.discovery_method.should == "rspec" }.to raise_error("Unknown discovery method rspec")
+ end
+
+ it "should only allow custom discovery methods if direct_addressing is enabled" do
+ @client.expects(:options).returns({:discovery_method => "rspec"}).twice
+ Config.instance.expects(:direct_addressing).returns(false)
+ @discovery.expects(:has_method?).with("rspec").returns(true)
+ expect { @discovery.discovery_method.should == "rspec" }.to raise_error("Custom discovery methods require direct addressing mode")
+ end
+ end
+ end
+end
View
4 spec/unit/logger/syslog_logger_spec.rb
@@ -10,8 +10,8 @@ module Logger
describe Syslog_logger do
describe "#start" do
before do
- Config.any_instance.stubs(:logfacility).returns("user")
- Config.any_instance.stubs(:loglevel).returns("error")
+ Config.instance.stubs(:logfacility).returns("user")
+ Config.instance.stubs(:loglevel).returns("error")
end
it "should close the syslog if already opened" do
View
31 spec/unit/message_spec.rb
@@ -67,7 +67,7 @@ module MCollective
describe "#reply_to=" do
it "should only set the reply-to header for requests" do
- Config.instance.instance_variable_set("@direct_addressing", true)
+ Config.instance.expects(:direct_addressing).returns(true)
m = Message.new("payload", "message", :type => :reply)
m.discovered_hosts = ["foo"]
expect { m.reply_to = "foo" }.to raise_error(/reply targets/)
@@ -146,7 +146,7 @@ module MCollective
describe "#type=" do
it "should only allow types to be set when discovered hosts were given" do
m = Message.new("payload", "message")
- Config.instance.instance_variable_set("@direct_addressing", true)
+ Config.instance.stubs(:direct_addressing).returns(true)
expect {
m.type = :direct_request
@@ -155,7 +155,7 @@ module MCollective
it "should not allow direct_request to be set if direct addressing isnt enabled" do
m = Message.new("payload", "message")
- Config.instance.instance_variable_set("@direct_addressing", false)
+ Config.instance.stubs(:direct_addressing).returns(false)
expect {
m.type = :direct_request
@@ -164,13 +164,24 @@ module MCollective
it "should only accept valid types" do
m = Message.new("payload", "message")
- Config.instance.instance_variable_set("@direct_addressing", true)
+ Config.instance.stubs(:direct_addressing).returns(true)
expect {
m.type = :foo
}.to raise_error("Unknown message type foo")
end
+ it "should clear the filter in direct_request mode and add just an agent filter" do
+ m = Message.new("payload", "message")
+ m.discovered_hosts = ["rspec"]
+ Config.instance.stubs(:direct_addressing).returns(true)
+
+ m.filter = Util.empty_filter.merge({"cf_class" => ["test"]})
+ m.agent = "rspec"
+ m.type = :direct_request
+ m.filter.should == Util.empty_filter.merge({"agent" => ["rspec"]})
+ end
+
it "should set the type" do
m = Message.new("payload", "message")
m.type = :request
@@ -201,7 +212,7 @@ module MCollective
security.expects(:encoderequest).with("identity", 'payload', '123', Util.empty_filter, 'rspec_agent', 'mcollective', 60).twice
PluginManager.expects("[]").with("security_plugin").returns(security).twice
- Config.any_instance.expects(:identity).returns("identity").times(4)
+ Config.instance.expects(:identity).returns("identity").times(4)
Message.any_instance.expects(:requestid).returns("123").twice
@@ -358,8 +369,8 @@ module MCollective
m = Message.new("msg", "message", :type => :request)
m.discovered_hosts = ["one", "two", "three"]
- Config.any_instance.expects(:direct_addressing).returns(true)
- Config.any_instance.expects(:direct_addressing_threshold).returns(10)
+ Config.instance.stubs(:direct_addressing).returns(true)
+ Config.instance.stubs(:direct_addressing_threshold).returns(10)
connector = mock
connector.expects(:publish).with(m)
@@ -373,8 +384,8 @@ module MCollective
m = Message.new("msg", "message", :type => :request)
m.discovered_hosts = ["one", "two", "three"]
- Config.any_instance.expects(:direct_addressing).returns(true)
- Config.any_instance.expects(:direct_addressing_threshold).returns(1)
+ Config.instance.expects(:direct_addressing).returns(true)
+ Config.instance.expects(:direct_addressing_threshold).returns(1)
connector = mock
connector.expects(:publish).with(m)
@@ -389,7 +400,7 @@ module MCollective
it "should create a valid request id" do
m = Message.new("msg", "message", :agent => "rspec", :collective => "mc")
- Config.any_instance.expects(:identity).returns("rspec")
+ Config.instance.expects(:identity).returns("rspec")
Time.expects(:now).returns(1.1)
Digest::MD5.expects(:hexdigest).with("rspec-1.1-rspec-mc").returns("reqid")
View
4 spec/unit/optionparser_spec.rb
@@ -97,15 +97,15 @@ module MCollective
parser = Optionparser.new(defaults={:collective => "rspec"})
parser.stubs(:add_required_options)
parser.stubs(:add_common_options)
- Config.any_instance.expects(:main_collective).never
+ Config.instance.expects(:main_collective).never
parser.parse
end
it "should set the active collective from the config class if not given on the cli" do
parser = Optionparser.new(defaults={})
parser.stubs(:add_required_options)
parser.stubs(:add_common_options)
- Config.any_instance.expects(:main_collective).returns(:rspec).once
+ Config.instance.expects(:main_collective).returns(:rspec).once
parser.parse[:collective].should == :rspec
end
end
View
48 spec/unit/plugins/mcollective/discovery/flatfile_spec.rb
@@ -0,0 +1,48 @@
+#!/usr/bin/env rspec
+
+require 'spec_helper'
+
+require File.dirname(__FILE__) + '/../../../../../plugins/mcollective/discovery/flatfile.rb'
+
+module MCollective
+ class Discovery
+ describe Flatfile do
+ describe "#discover" do
+ before do
+ @client = mock
+ @client.stubs(:options).returns({})
+ @client.stubs(:options).returns({:discovery_options => ["/nonexisting"]})
+
+
+ File.stubs(:readable?).with("/nonexisting").returns(true)
+ File.stubs(:readlines).with("/nonexisting").returns(["one", "two"])
+ end
+
+ it "should use a file specified in discovery_options" do
+ File.expects(:readable?).with("/nonexisting").returns(true)
+ File.expects(:readlines).with("/nonexisting").returns(["one", "two"])
+ Flatfile.discover(Util.empty_filter, 0, 0, @client).should == ["one", "two"]
+ end
+
+ it "should fail unless a file is specified" do
+ @client.stubs(:options).returns({:discovery_options => []})
+ expect { Flatfile.discover(Util.empty_filter, 0, 0, @client) }.to raise_error("The flatfile discovery method needs a path to a text file")
+ end
+
+ it "should fail for unreadable files" do
+ File.expects(:readable?).with("/nonexisting").returns(false)
+
+ expect { Flatfile.discover(Util.empty_filter, 0, 0, @client) }.to raise_error("Cannot read the file /nonexisting specified as discovery source")
+ end
+
+ it "should regex filters" do
+ Flatfile.discover(Util.empty_filter.merge("identity" => [/one/]), 0, 0, @client).should == ["one"]
+ end
+
+ it "should filter against non regex nodes" do
+ Flatfile.discover(Util.empty_filter.merge("identity" => ["one"]), 0, 0, @client).should == ["one"]
+ end
+ end
+ end
+ end
+end
View
40 spec/unit/plugins/mcollective/discovery/mc_spec.rb
@@ -0,0 +1,40 @@
+#!/usr/bin/env rspec
+
+require 'spec_helper'
+
+require File.dirname(__FILE__) + '/../../../../../plugins/mcollective/discovery/mc.rb'
+
+module MCollective
+ class Discovery
+ describe Mc do
+ describe "#discover" do
+ before do
+ @reply = mock
+ @reply.stubs(:payload).returns({:senderid => "rspec"})
+
+ @client = mock
+ @client.stubs(:sendreq)
+ @client.stubs(:unsubscribe)
+ @client.stubs(:receive).returns(@reply)
+
+ Log.stubs(:debug)
+ end
+
+ it "should send the ping request via the supplied client" do
+ @client.expects(:sendreq).with("ping", "discovery", Util.empty_filter).returns("123456")
+ Mc.discover(Util.empty_filter, 1, 1, @client)
+ end
+
+ it "should stop early if a limit is supplied" do
+ @client.stubs(:receive).returns(@reply).times(10)
+ Mc.discover(Util.empty_filter, 1, 10, @client).should == ("rspec," * 10).split(",")
+ end
+
+ it "should unsubscribe from the discovery reply source" do
+ @client.expects(:unsubscribe).with("discovery", :reply)
+ Mc.discover(Util.empty_filter, 1, 10, @client).should == ("rspec," * 10).split(",")
+ end
+ end
+ end
+ end
+end
View
4 spec/unit/rpc/actionrunner_spec.rb
@@ -185,7 +185,7 @@ module RPC
end
it "should find the first match in the libdir" do
- Config.any_instance.expects(:libdir).returns(["#{File::SEPARATOR}libdir1", "#{File::SEPARATOR}libdir2"])
+ Config.instance.expects(:libdir).returns(["#{File::SEPARATOR}libdir1", "#{File::SEPARATOR}libdir2"])