@@ -33,7 +33,7 @@ module DeepTest def deploy_agents @agents_deployed = true super - @warlock.exit_when_none_running + warlock.exit_when_none_running end def agents_deployed? @@ -43,18 +43,16 @@ module DeepTest def daemonize(address, grace_period = MERCY_KILLING_GRACE_PERIOD) innie, outie = IO.pipe - self.class.warlock.start("Beachhead") do + warlock.start("Beachhead", :detach_io => true) do innie.close - DRb.start_service("drubyall://#{address}:0", self) + DRb.start_service "drubyall://#{address}:0", self DeepTest.logger.info { "Beachhead started at #{DRb.uri}" } outie.write DRb.uri outie.close - launch_mercy_killer(grace_period) - - yield if block_given? + launch_mercy_killer grace_period DRb.thread.join end @@ -62,11 +60,7 @@ module DeepTest outie.close uri = innie.gets innie.close - DRbObject.new_with_uri(uri) - end - - def self.warlock - @warlock ||= DeepTest::Warlock.new + DRbObject.new_with_uri uri end end end lib/deep_test/distributed/beachhead.rb @@ -13,12 +13,13 @@ module DeepTest end def establish_beachhead(options) - output = `#{ssh_command(options)} '#{spawn_command(options)}'` + output = `#{ssh_command(options)} '#{spawn_command(options)}' 2>&1` output.each do |line| if line =~ /Beachhead url: (.*)/ return DRb::DRbObject.new_with_uri($1) end end + raise "LandingShip unable to establish Beachhead. Output from #{@config[:address]} was:\n#{output}" end def ssh_command(options) lib/deep_test/distributed/landing_ship.rb @@ -1,11 +1,16 @@ module DeepTest class LocalDeployment + attr_reader :warlock + def initialize(options, agent_class = DeepTest::Agent) @options = options - @warlock = Warlock.new @agent_class = agent_class end + def warlock + @warlock ||= Warlock.new central_command + end + def load_files(files) files.each {|f| load f} end @@ -17,22 +22,16 @@ module DeepTest def deploy_agents each_agent do |agent_num| start_agent(agent_num) do - ProxyIO.replace_stdout_stderr!(central_command.stdout, central_command.stderr) do - reseed_random_numbers - reconnect_to_database - agent = @agent_class.new(agent_num, central_command, @options.new_listener_list) - agent.run - end + reseed_random_numbers + reconnect_to_database + agent = @agent_class.new(agent_num, central_command, @options.new_listener_list) + agent.run end end end def terminate_agents - @warlock.stop_demons - end - - def wait_for_completion - @warlock.wait_for_all_to_finish + warlock.stop_demons end def number_of_agents @@ -46,7 +45,7 @@ module DeepTest end def start_agent(agent_num, &blk) - @warlock.start("agent #{agent_num}", &blk) + warlock.start("agent #{agent_num}", &blk) end def reseed_random_numbers lib/deep_test/local_deployment.rb @@ -6,6 +6,10 @@ module DeepTest self.level = configured_log_level end + def io_stream + @logdev.dev + end + def configured_log_level if ENV['DEEP_TEST_LOG_LEVEL'] Logger.const_get(ENV['DEEP_TEST_LOG_LEVEL']) lib/deep_test/logger.rb @@ -1,27 +1,27 @@ module DeepTest class Main - def self.run(options, deployment, runner) - new(options, deployment, runner).run - end - - def initialize(options, deployment, runner) + def initialize(options, deployment, runner, central_command = nil) @options = options - @runner = runner @deployment = deployment + @runner = runner + @central_command = central_command || CentralCommand.start(options) + end + + def load_files(files) + @deployment.load_files files end def run(exit_when_done = true) passed = false begin - central_command = CentralCommand.start(@options) @options.new_listener_list.before_starting_agents @deployment.deploy_agents begin DeepTest.logger.debug { "Loader Starting (#{$$})" } passed = @runner.process_work_units ensure - shutdown(central_command) + shutdown end ensure DeepTest.logger.debug { "Main: Stopping CentralCommand" } @@ -31,9 +31,9 @@ module DeepTest Kernel.exit(passed ? 0 : 1) if exit_when_done end - def shutdown(central_command) + def shutdown DeepTest.logger.debug { "Main: Shutting Down" } - central_command.done_with_work + @central_command.done_with_work first_exception = $! begin lib/deep_test/main.rb @@ -15,6 +15,10 @@ module DeepTest @target.flush end + def reopen(*args) + @target.reopen *args + end + def self.replace_stdout_stderr!(new_stdout, new_stderr) old_stdout_const, old_stdout_global = STDOUT, $stdout old_stderr_const, old_stderr_global = STDERR, $stderr lib/deep_test/proxy_io.rb @@ -20,14 +20,10 @@ task :'deep_test:establish_beachhead' do beachhead = DeepTest::Distributed::Beachhead.new( options.mirror_path('/tmp'), options, DeepTest::Distributed::SshClientConnectionInfo.new - ).daemonize(ENV['HOST']) do - STDOUT.reopen("/tmp/deep_test_server.log", "a") - STDERR.reopen(STDOUT) - end + ).daemonize(ENV['HOST']) puts "Beachhead url: #{beachhead.__drburi}" + $stdout.flush - STDOUT.close - STDERR.close exit!(0) end lib/deep_test/rake_tasks.rb @@ -9,9 +9,9 @@ module DeepTest @runner_options = options # added to make work with 1.1.12 end @deep_test_options = DeepTest::Options.from_command_line(deep_test_options) - DeepTest.init(@deep_test_options) + DeepTest.init @deep_test_options + @main = Main.new @deep_test_options, @deep_test_options.new_deployment, self, central_command @central_command = central_command - @deployment = @deep_test_options.new_deployment end def central_command @@ -22,11 +22,11 @@ module DeepTest end def load_files(files) - @deployment.load_files files + @main.load_files files end def run - Main.run(@deep_test_options, @deployment, self) + @main.run end def process_work_units lib/deep_test/spec/runner.rb @@ -1,12 +1,13 @@ module DeepTest class Warlock - def initialize + def initialize(central_command) + @central_command = central_command @demons_semaphore = Mutex.new @demons = [] @reapers = [] end - def start(name, &block) + def start(name, options = {}, &block) # Not synchronizing for the fork seems to cause # random errors (Bus Error, Segfault, and GC non-object) # in Beachhead processes. @@ -15,23 +16,30 @@ module DeepTest pid = nil @demons_semaphore.synchronize do pid = DeepTest.drb_safe_fork do - catch(:exit_demon) do - Signal.trap("TERM") { throw :exit_demon } - - # Fork leaves the semaphore locked and we'll never make it - # to end of synchronize block. - # - # The Ruby 1.8.6 C mutex implementation automatically treats - # a mutex locked by a dead thread as unlocked and will raise - # an error if we try to unlock it from this thread. - # - @demons_semaphore.unlock if @demons_semaphore.locked? - - begin - yield - rescue Exception => e - DeepTest.logger.debug { "Exception in #{name} (#{Process.pid}): #{e.message}" } - raise + if options[:detach_io] + $stdout.reopen("/dev/null") + $stderr.reopen("/dev/null") + end + + ProxyIO.replace_stdout_stderr!(@central_command.stdout, @central_command.stderr) do + catch(:exit_demon) do + Signal.trap("TERM") { throw :exit_demon } + + # Fork leaves the semaphore locked and we'll never make it + # to end of synchronize block. + # + # The Ruby 1.8.6 C mutex implementation automatically treats + # a mutex locked by a dead thread as unlocked and will raise + # an error if we try to unlock it from this thread. + # + @demons_semaphore.unlock if @demons_semaphore.locked? + + begin + yield + rescue Exception => e + DeepTest.logger.debug { "Exception in #{name} (#{Process.pid}): #{e.message}" } + raise + end end end @@ -80,6 +88,7 @@ module DeepTest def exit_when_none_running Thread.new do wait_for_all_to_finish + DeepTest.logger.debug { "exiting #{Process.pid} with all demons finished" } exit(0) end end lib/deep_test/warlock.rb @@ -1,7 +1,6 @@ require File.dirname(__FILE__) + "/../../lib/deep_test" options = DeepTest::Options.from_command_line(ARGV[0]) DeepTest.init(options) -runner = DeepTest::Test::Runner.new(options) -deployment = options.new_deployment -deployment.load_files Dir.glob(options.pattern) -DeepTest::Main.run(options, deployment, runner) +main = DeepTest::Main.new options, options.new_deployment, DeepTest::Test::Runner.new(options) +main.load_files Dir.glob(options.pattern) +main.run script/internal/run_test_suite.rb @@ -25,41 +25,47 @@ module DeepTest test "service is removed after grace period if agents haven't been started" do log_level = DeepTest.logger.level - begin - DeepTest.logger.level = Logger::ERROR - beachhead = Beachhead.new("", Options.new({}), stub_everything) - beachhead.stubs(:start_agent) - beachhead.daemonize("localhost", 0.25) - # Have to sleep long enough to warlock to reap dead process - sleep 1.0 - assert_equal 0, Beachhead.warlock.demon_count - ensure + SimpleTestCentralCommand.new.with_drb_server do |central_command| begin - Beachhead.warlock.stop_demons + DeepTest.logger.level = Logger::ERROR + beachhead = Beachhead.new("", Options.new({}), stub_everything) + beachhead.stubs(:central_command).returns central_command + beachhead.stubs(:start_agent) + beachhead.daemonize("localhost", 0.25) + # Have to sleep long enough to warlock to reap dead process + sleep 1.0 + assert_equal 0, beachhead.warlock.demon_count ensure - DeepTest.logger.level = log_level + begin + beachhead.warlock.stop_demons + ensure + DeepTest.logger.level = log_level + end end end end test "service is not removed after grace period if agents have been started" do log_level = DeepTest.logger.level - begin - DeepTest.logger.level = Logger::ERROR - beachhead = Beachhead.new("", Options.new({}), stub_everything) - beachhead.stubs(:start_agent) - # Since we're not actually starting agents, we don't want to exit when none are running for this test - beachhead.instance_variable_get(:@warlock).stubs(:exit_when_none_running) - remote_reference = beachhead.daemonize("localhost", 0.25) - remote_reference.deploy_agents - # Have to sleep long enough to warlock to reap dead process - sleep 1.0 - assert_equal 1, Beachhead.warlock.demon_count - ensure + SimpleTestCentralCommand.new.with_drb_server do |central_command| begin - Beachhead.warlock.stop_demons + DeepTest.logger.level = Logger::ERROR + beachhead = Beachhead.new "", Options.new({}), stub(:address => "localhost") + beachhead.stubs(:central_command).returns central_command + beachhead.stubs(:start_agent) + # Since we're not actually starting agents, we don't want to exit when none are running for this test + beachhead.warlock.stubs(:exit_when_none_running) + remote_reference = beachhead.daemonize("localhost", 0.25) + remote_reference.deploy_agents + # Have to sleep long enough to warlock to reap dead process + sleep 1.0 + assert_equal 1, beachhead.warlock.demon_count ensure - DeepTest.logger.level = log_level + begin + beachhead.warlock.stop_demons + ensure + DeepTest.logger.level = log_level + end end end end test/deep_test/distributed/beachhead_test.rb @@ -112,7 +112,7 @@ module DeepTest assert_raises(NoDispatchReceiversError) {controller.dispatch(:another_call)} end - test "error is printed with backtrace when it occurr" do + test "error is printed with backtrace when it occurrs" do e = Exception.new("message") e.set_backtrace %w[file1:1 file2:2] receiver = mock test/deep_test/distributed/dispatch_controller_test.rb @@ -20,7 +20,7 @@ module DeepTest "ssh -4 remote_host " + "'#{ShellEnvironment.like_login} && cd /tmp/myhost_my_local_dir && " + "rake deep_test:establish_beachhead " + - "OPTIONS=#{options.to_command_line} HOST=remote_host'" + "OPTIONS=#{options.to_command_line} HOST=remote_host' 2>&1" ).returns("blah blah\nBeachhead url: druby://remote_host:9999\nblah") beachhead = landing_ship.establish_beachhead(options) @@ -37,7 +37,7 @@ module DeepTest "ssh -4 remote_host -l me " + "'#{ShellEnvironment.like_login} && cd /tmp/myhost_my_local_dir && " + "rake deep_test:establish_beachhead " + - "OPTIONS=#{options.to_command_line} HOST=remote_host'" + "OPTIONS=#{options.to_command_line} HOST=remote_host' 2>&1" ).returns("blah blah\nBeachhead url: druby://remote_host:9999\nblah") landing_ship.establish_beachhead(options) test/deep_test/distributed/landing_ship_test.rb @@ -20,43 +20,5 @@ module DeepTest deployment.load_files([:file_1, :file_2]) end - - test "deploy_agents redirects stdout and stderr back to central_command" do - agent_class = Class.new do - def initialize(agent_num, central_command, listeners); end - def run; puts "hello stdout"; $stderr.puts "hello stderr" end - end - - central_command = stub :stdout => StringIO.new, :stderr => StringIO.new - - with_drb_server_for central_command do |drb_server| - options = stub :number_of_agents => 1, - :central_command => DRbObject.new_with_uri(drb_server.uri), - :new_listener_list => [] - - run_agents_to_completion LocalDeployment.new(options, agent_class) - end - - assert_equal "hello stdout\n", central_command.stdout.string - assert_equal "hello stderr\n", central_command.stderr.string - end - - def with_drb_server_for(front) - # using drbunix prevents a getaddrinfo on our host, which can take 5 seconds - drb_server = DRb::DRbServer.new "drbunix:", front - - begin - yield drb_server - ensure - drb_server.stop_service - end - end - - def run_agents_to_completion(deployment) - deployment.deploy_agents - deployment.wait_for_completion - ensure - deployment.terminate_agents - end end end test/deep_test/local_deployment_test.rb @@ -3,11 +3,10 @@ require File.dirname(__FILE__) + "/../test_helper" module DeepTest unit_tests do test "shutdown calls done_with_work" do - main = Main.new(nil, stub_everything, nil) - central_command = mock + main = Main.new(nil, stub_everything, nil, central_command = mock) central_command.expects(:done_with_work) - main.shutdown(central_command) + main.shutdown end end end test/deep_test/main_test.rb @@ -3,38 +3,105 @@ require File.dirname(__FILE__) + '/../test_helper' module DeepTest unit_tests do test "running? is true if sending kill(0, pid) does not fail" do - warlock = Warlock.new + warlock = Warlock.new mock Process.expects(:kill).with(0, :pid) assert_equal true, warlock.running?(:pid) end test "running? is false if Process.kill(0, pid) raises Errno::ESRCH" do - warlock = Warlock.new + warlock = Warlock.new mock Process.stubs(:kill).raises(Errno::ESRCH) assert_equal false, warlock.running?(:pid) end test "running? is true if Process.kill raises Exception" do - warlock = Warlock.new + warlock = Warlock.new mock Process.stubs(:kill).raises(Exception) assert_equal true, warlock.running?(:pid) end test "demon_count is 0 initially" do - assert_equal 0, Warlock.new.demon_count + assert_equal 0, Warlock.new(mock).demon_count end test "add_demon increases demon_count by 1" do - warlock = Warlock.new + warlock = Warlock.new mock warlock.send(:add_demon, "name", 1) assert_equal 1, warlock.demon_count end test "remove_demon increases demon_count by 1" do - warlock = Warlock.new + warlock = Warlock.new mock warlock.send(:add_demon, "name", 1) warlock.send(:remove_demon, "name", 1) assert_equal 0, warlock.demon_count end + + test "start redirects stdout and stderr back to central_command" do + central_command = SimpleTestCentralCommand.new + central_command.with_drb_server do |remote_reference| + warlock = Warlock.new remote_reference + begin + warlock.start("test") do + puts "hello stdout" + $stderr.puts "hello stderr" + end + warlock.wait_for_all_to_finish + ensure + warlock.stop_demons + end + end + + assert_equal "hello stdout\n", central_command.stdout.string + assert_equal "hello stderr\n", central_command.stderr.string + end + + test "start reopens original stdout and stderr to /dev/null if detach_io is true" do + test_stdout, test_stderr = Tempfile.new("stdout"), Tempfile.new("stderr") + central_command = SimpleTestCentralCommand.new + central_command.with_drb_server do |remote_reference| + warlock = Warlock.new remote_reference + begin + require 'tempfile' + ProxyIO.replace_stdout_stderr! test_stdout, test_stderr do + warlock.start("test", :detach_io => true) do + test_stdout.puts "hello stdout" + test_stderr.puts "hello stderr" + end + warlock.wait_for_all_to_finish + end + ensure + warlock.stop_demons + end + end + + test_stdout.rewind; test_stderr.rewind + assert_equal "", test_stdout.read + assert_equal "", test_stderr.read + end + + test "start leave original stdout and untouched if detach_io is false" do + test_stdout, test_stderr = Tempfile.new("stdout"), Tempfile.new("stderr") + central_command = SimpleTestCentralCommand.new + central_command.with_drb_server do |remote_reference| + warlock = Warlock.new remote_reference + begin + require 'tempfile' + ProxyIO.replace_stdout_stderr! test_stdout, test_stderr do + warlock.start("test", :detach_io => false) do + test_stdout.puts "hello stdout" + test_stderr.puts "hello stderr" + end + warlock.wait_for_all_to_finish + end + ensure + warlock.stop_demons + end + end + + test_stdout.rewind; test_stderr.rewind + assert_equal "hello stdout\n", test_stdout.read + assert_equal "hello stderr\n", test_stderr.read + end end end test/deep_test/warlock_test.rb @@ -1,11 +1,25 @@ module DeepTest class SimpleTestCentralCommand attr_accessor :debug, :simulate_result_overdue_error + attr_accessor :stdout, :stderr def initialize @work_units = [] @test_results = [] @semaphore = Mutex.new + @stdout = StringIO.new + @stderr = StringIO.new + end + + def with_drb_server + # using drbunix prevents a getaddrinfo on our host, which can take 5 seconds + drb_server = DRb::DRbServer.new "drbunix:", self + + begin + yield DRbObject.new_with_uri(drb_server.uri) + ensure + drb_server.stop_service + end end def take_result test/simple_test_central_command.rb @@ -1,10 +1,10 @@ class TestLogger < DeepTest::Logger - def initialize + def initialize(device = nil) super(StringIO.new) end def logged_output - @logdev.dev.string + io_stream.string end end test/test_logger.rb 69cc204df6072430103359f2622bb8a0494d90a8 qxjit (David Vollbracht) david.vollbracht@gmail.com http://github.com/qxjit/deep-test/commit/b75f3af1c831040ff97cef80f9217b2a89056daa b75f3af1c831040ff97cef80f9217b2a89056daa 2009-06-29T19:30:07-07:00 2009-06-29T19:30:07-07:00 push redirecting of stdout and stderr back to central command down into warlock 85afe014e50716d6a8895009aa527e8fbb1f8c05 qxjit (David Vollbracht) david.vollbracht@gmail.com