init/god test/configs/task/logs/.placeholder test/test_conditions_http_response_code.rb @@ -6,6 +6,7 @@ * Implement notification system * Add Tasks (a generalization of Watches) to do non-process related tasks * Add example init.d file in GOD_INSTALL_DIR/init/god + * Add human readable info to conditions (and make low level log lines debug) * Minor Enchancements * Allow EventConditions to do transition overloading * Report errors during god startup instead of failing silently @@ -17,6 +18,7 @@ * Add `god quit` to stop god without stopping any tasks * New Conditions * Flapping < TriggerCondition - trigger on state change + * HttpResponseCode < PollCondition - trigger on http response code or timeout (thx scott becker) * New Contacts * Email < Contact - notify via email (smtp, sendmail) * Bug Fixes History.txt @@ -18,6 +18,7 @@ lib/god/conditions/always.rb lib/god/conditions/cpu_usage.rb lib/god/conditions/degrading_lambda.rb lib/god/conditions/flapping.rb +lib/god/conditions/http_response_code.rb lib/god/conditions/lambda.rb lib/god/conditions/memory_usage.rb lib/god/conditions/process_exits.rb @@ -62,12 +63,14 @@ test/configs/real.rb test/configs/running_load/running_load.god test/configs/stress/simple_server.rb test/configs/stress/stress.god +test/configs/task/logs/.placeholder test/configs/task/task.god test/configs/test.rb test/helper.rb test/suite.rb test/test_behavior.rb test/test_condition.rb +test/test_conditions_http_response_code.rb test/test_conditions_process_running.rb test/test_conditions_tries.rb test/test_contact.rb Manifest.txt @@ -73,8 +73,9 @@ module God VERSION = '0.5.0' LOG = Logger.new + LOG.datetime_format = "%Y-%m-%d %H:%M:%S " - LOG_BUFFER_SIZE_DEFAULT = 100 + LOG_BUFFER_SIZE_DEFAULT = 1000 PID_FILE_DIRECTORY_DEFAULT = '/var/run/god' DRB_PORT_DEFAULT = 17165 DRB_ALLOW_DEFAULT = ['127.0.0.1'] @@ -125,6 +126,7 @@ module God self.pid_file_directory ||= PID_FILE_DIRECTORY_DEFAULT self.port ||= DRB_PORT_DEFAULT self.allow ||= DRB_ALLOW_DEFAULT + LOG.level = Logger::INFO # init has been executed self.inited = true lib/god.rb @@ -1,7 +1,7 @@ module God class Condition < Behavior - attr_accessor :transition, :notify + attr_accessor :transition, :notify, :info # Generate a Condition of the given kind. The proper class if found by camel casing the # kind (which is given as an underscored symbol). lib/god/condition.rb @@ -4,6 +4,10 @@ module God class Always < PollCondition attr_accessor :what + def initialize + self.info = "always" + end + def valid? valid = true valid &= complain("Attribute 'what' must be specified", self) if self.what.nil? lib/god/conditions/always.rb @@ -31,10 +31,15 @@ module God pid = File.read(self.watch.pid_file).strip process = System::Process.new(pid) @timeline.push(process.percent_cpu) + + history = "[" + @timeline.map { |x| "#{x}%%" }.join(", ") + "]" + if @timeline.select { |x| x > self.above }.size >= self.times.first @timeline.clear + self.info = "cpu out of bounds #{history}" return true else + self.info = "cpu within bounds #{history}" return false end end lib/god/conditions/cpu_usage.rb @@ -4,6 +4,10 @@ module God class Flapping < TriggerCondition attr_accessor :times, :within, :from_state, :to_state, :retry_in, :retry_times, :retry_within + def initialize + self.info = "process is flapping" + end + def prepare @timeline = Timeline.new(self.times) @retry_timeline = Timeline.new(self.retry_times) lib/god/conditions/flapping.rb @@ -4,16 +4,22 @@ module God module Conditions class HttpResponseCode < PollCondition - attr_accessor :code_is, :code_is_not, :times, :host, :port, :timeout, :path - + attr_accessor :code_is, # e.g. 500 or '500' or [404, 500] or %w{404 500} + :code_is_not, # e.g. 200 or '200' or [200, 302] or %w{200 302} + :times, # e.g. 3 or [3, 5] + :host, # e.g. www.example.com + :port, # e.g. 8080 + :timeout, # e.g. 60.seconds + :path # e.g. '/' + def initialize super self.times = [1, 1] end def prepare - self.code_is = Array(self.code) if self.code_is - self.code_is_not = Array(self.code_is_not) if self.code_is_not + self.code_is = Array(self.code_is).map { |x| x.to_i } if self.code_is + self.code_is_not = Array(self.code_is_not).map { |x| x.to_i } if self.code_is_not if self.times.kind_of?(Integer) self.times = [self.times, self.times] @@ -21,7 +27,7 @@ module God @timeline = Timeline.new(self.times[1]) end - + def valid? valid = true valid &= complain("Attribute 'host' must be specified", self) if self.host.nil? @@ -32,7 +38,7 @@ module God valid &= complain("Attribute 'timeout' must be specified", self) if self.timeout.nil? valid end - + def test response = nil @@ -41,22 +47,33 @@ module God response = http.head(self.path) end - if self.code_is && self.code_is.include?(response.code) + actual_response_code = response.code.to_i + if self.code_is && self.code_is.include?(actual_response_code) pass - elsif self.code_is_not && !self.code.include?(response.code) + elsif self.code_is_not && !self.code_is_not.include?(actual_response_code) pass else - false + fail end rescue Timeout::Error - self.code_is ? false : pass + self.code_is ? fail : pass end private def pass - @timeline.clear - return true + @timeline << true + if @timeline.select { |x| x }.size >= self.times.first + @timeline.clear + true + else + false + end + end + + def fail + @timeline << false + false end end lib/god/conditions/http_response_code.rb @@ -11,7 +11,13 @@ module God end def test - self.lambda.call() + if self.lambda.call() + self.info = "lambda condition was satisfied" + true + else + self.info = "lambda condition was not satisfied" + false + end end end lib/god/conditions/lambda.rb @@ -31,10 +31,15 @@ module God pid = File.read(self.watch.pid_file).strip process = System::Process.new(pid) @timeline.push(process.memory) + + history = "[" + @timeline.map { |x| "#{x}kb" }.join(", ") + "]" + if @timeline.select { |x| x > self.above }.size >= self.times.first @timeline.clear + self.info = "memory out of bounds #{history}" return true else + self.info = "memory within bounds #{history}" return false end end lib/god/conditions/memory_usage.rb @@ -2,6 +2,10 @@ module God module Conditions class ProcessExits < EventCondition + def initialize + self.info = "process exited" + end + def valid? valid = true valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? lib/god/conditions/process_exits.rb @@ -12,18 +12,30 @@ module God end def test + self.info = [] + unless File.exist?(self.watch.pid_file) - msg = "#{self.watch.name} #{self.class.name}: no such pid file: #{self.watch.pid_file}" - Syslog.debug(msg) - LOG.log(self.watch, :info, msg) - + self.info << "#{self.watch.name} #{self.class.name}: no such pid file: #{self.watch.pid_file}" return !self.running end pid = File.read(self.watch.pid_file).strip active = System::Process.new(pid).exists? - (self.running && active) || (!self.running && !active) + if (self.running && active) + self.info << "process is running" + true + elsif (!self.running && !active) + self.info << "process is not running" + true + else + if self.running + self.info << "process is not running" + else + self.info << "process is running" + end + false + end end end lib/god/conditions/process_running.rb @@ -20,10 +20,14 @@ module God concensus = (@timeline.size == self.times) duration = self.within.nil? || (@timeline.last - @timeline.first) < self.within + history = "[" + @timeline.map { |x| "#{x}" }.join(", ") + "]" + if concensus && duration @timeline.clear if within.nil? + self.info = "tries exceeded #{history}" return true else + self.info = "tries within bounds #{history}" return false end end lib/god/conditions/tries.rb @@ -26,10 +26,14 @@ module God true end - def friendly_name + def base_name self.class.name.split('::').last end + def friendly_name + base_name + end + def self.complain(text, c = nil) msg = text msg += " for #{c.friendly_name}" if c lib/god/configurable.rb @@ -55,9 +55,7 @@ module God result = condition.test # log - msg = watch.name + ' ' + condition.class.name + " [#{result}] " + self.dest_desc(metric, condition) - Syslog.debug(msg) - LOG.log(watch, :info, msg) + self.log(watch, metric, condition, result) # notify if condition.notify @@ -113,9 +111,8 @@ module God watch = metric.watch watch.mutex.synchronize do - msg = watch.name + ' ' + condition.class.name + " [true] " + self.dest_desc(metric, condition) - Syslog.debug(msg) - LOG.log(watch, :info, msg) + # log + self.log(watch, metric, condition, true) # notify if condition.notify @@ -142,12 +139,44 @@ module God # helpers + def self.log(watch, metric, condition, result) + # log info if available + if condition.info + begin + status = + if (metric.destination && metric.destination.keys.size == 2) || result == true + "[trigger]" + else + "[ok]" + end + + Array(condition.info).each do |condition_info| + msg = "#{watch.name} #{status} #{condition_info} (#{condition.base_name})" + Syslog.debug(msg) + LOG.log(watch, :info, msg % []) + end + rescue Exception => e + puts e.message + puts e.backtrace.join("\n") + end + else + msg = "#{watch.name} [unknown] (#{condition.base_name})" + Syslog.debug(msg) + LOG.log(watch, :info, msg % []) + end + + # log + msg = watch.name + ' ' + condition.base_name + " [#{result}] " + self.dest_desc(metric, condition) + Syslog.debug(msg) + LOG.log(watch, :debug, msg) + end + def self.dest_desc(metric, condition) - if metric.destination - metric.destination.inspect + if condition.transition + {true => condition.transition}.inspect else - if condition.transition - {true => condition.transition}.inspect + if metric.destination + metric.destination.inspect else 'none' end lib/god/hub.rb @@ -10,6 +10,18 @@ God.watch do |w| end end + w.restart_if do |restart| + restart.condition(:cpu_usage) do |c| + c.above = 50.percent + c.times = [3, 5] + end + + restart.condition(:memory_usage) do |c| + c.above = 10.megabytes + c.times = [3, 5] + end + end + # lifecycle w.lifecycle do |on| on.condition(:flapping) do |c| @@ -21,10 +33,5 @@ God.watch do |w| c.retry_times = 2 c.retry_within = 5.minutes end - - on.condition(:cpu_usage) do |c| - c.above = 10.percent - c.times = [3, 5] - end end end \ No newline at end of file test/configs/child_polls/child_polls.god @@ -1,3 +1,12 @@ #! /usr/bin/env ruby -loop { STDOUT.puts('server'); STDOUT.flush; sleep 1 } \ No newline at end of file +data = '' + +loop do + STDOUT.puts('server'); + STDOUT.flush; + + 100000.times { data << 'x' } + + sleep 0.1 +end \ No newline at end of file test/configs/child_polls/simple_server.rb test/test_http_response_code.rb 889de6651b36d3eba10ac4afd6ea40eaa1aa84b5 Tom Werner tom@tom.desk.hq.powerset.com http://github.com/mojombo/god/commit/6dcfaa880de4efdd196a99c0bbbb1e21c4685aa3 6dcfaa880de4efdd196a99c0bbbb1e21c4685aa3 2007-09-20T18:12:09-07:00 2007-09-20T18:12:09-07:00 test http_response_code condition, add human readable condition logging 879cda8af0cb19c6a4484b8f07f781124bef3a06 Tom Werner tom@tom.desk.hq.powerset.com