Here is what we want to enable us to support simple cov

Capfile

@@ -0,0 +1,28 @@
+INSTANCES = %w[
+  ec2-184-73-185-51.compute-1.amazonaws.com
+  ec2-174-129-87-249.compute-1.amazonaws.com
+  ec2-174-129-137-32.compute-1.amazonaws.com
+  ec2-23-20-95-13.compute-1.amazonaws.com
+  ec2-23-20-123-107.compute-1.amazonaws.com
+  ec2-184-72-166-179.compute-1.amazonaws.com
+  ec2-23-20-9-246.compute-1.amazonaws.com
+  ec2-184-73-24-209.compute-1.amazonaws.com
+  ec2-23-20-146-206.compute-1.amazonaws.com
+  ec2-23-20-219-206.compute-1.amazonaws.com
+  ec2-50-17-115-219.compute-1.amazonaws.com
+  ec2-204-236-255-89.compute-1.amazonaws.com
+  ec2-184-72-165-139.compute-1.amazonaws.com
+  ec2-23-20-53-116.compute-1.amazonaws.com
+  ec2-50-16-63-50.compute-1.amazonaws.com
+  ec2-23-20-195-14.compute-1.amazonaws.com
+  ec2-23-20-214-168.compute-1.amazonaws.com
+  ec2-184-73-120-68.compute-1.amazonaws.com
+  ec2-184-72-208-156.compute-1.amazonaws.com
+  ec2-50-16-9-133.compute-1.amazonaws.com
+]
+
+role :workspaces, *INSTANCES
+
+task :kill_processes, :roles => :workspaces do
+  run "ps aux | grep -P 'cucumber|rspec|firefox|chrome|searchd|hobson' | grep -vE 'grep ' | awk '{print $2}' | xargs sudo kill -9"
+end

Gemfile.lock

@@ -32,6 +32,7 @@ PATH
       redis-slave
       resque (~> 1.19.0)
       sass
+      simplecov
       sinatra
       thor (~> 0.14.6)
       uuid
@@ -41,22 +42,22 @@ GEM
   remote: http://rubygems.org/
   specs:
     Platform (0.4.0)
-    actionpack (3.1.3)
-      activemodel (= 3.1.3)
-      activesupport (= 3.1.3)
+    actionpack (3.1.4)
+      activemodel (= 3.1.4)
+      activesupport (= 3.1.4)
       builder (~> 3.0.0)
       erubis (~> 2.7.0)
       i18n (~> 0.6)
-      rack (~> 1.3.5)
+      rack (~> 1.3.6)
       rack-cache (~> 1.1)
       rack-mount (~> 0.8.2)
       rack-test (~> 0.6.1)
       sprockets (~> 2.0.3)
-    activemodel (3.1.3)
-      activesupport (= 3.1.3)
+    activemodel (3.1.4)
+      activesupport (= 3.1.4)
       builder (~> 3.0.0)
       i18n (~> 0.6)
-    activesupport (3.1.3)
+    activesupport (3.1.4)
       multi_json (~> 1.0)
     archive-tar-minitar (0.5.2)
     builder (3.0.0)
@@ -65,17 +66,17 @@ GEM
     columnize (0.3.6)
     diff-lcs (1.1.3)
     erubis (2.7.0)
-    excon (0.6.6)
+    excon (0.13.3)
     ffi (1.0.11)
-    fog (0.9.0)
+    fog (1.3.1)
       builder
-      excon (~> 0.6.1)
-      formatador (>= 0.1.3)
-      json
+      excon (~> 0.13.0)
+      formatador (~> 0.2.0)
       mime-types
-      net-scp (>= 1.0.4)
-      net-ssh (>= 2.1.4)
-      nokogiri (>= 1.4.4)
+      multi_json (~> 1.0)
+      net-scp (~> 1.0.4)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
       ruby-hmac
     formatador (0.2.1)
     haml (3.1.4)
@@ -87,18 +88,18 @@ GEM
     log4r (1.1.10)
     macaddr (1.5.0)
       systemu (>= 2.4.0)
-    mime-types (1.17.2)
-    multi_json (1.1.0)
+    mime-types (1.18)
+    multi_json (1.2.0)
     net-scp (1.0.4)
       net-ssh (>= 1.99.1)
     net-ssh (2.3.0)
-    nokogiri (1.5.0)
+    nokogiri (1.5.2)
     open4 (1.3.0)
     popen4 (0.1.2)
       Platform (>= 0.4.0)
       open4 (>= 0.4.0)
     rack (1.3.6)
-    rack-cache (1.1)
+    rack-cache (1.2)
       rack (>= 0.4)
     rack-mount (0.8.3)
       rack (>= 1.0.0)
@@ -150,6 +151,10 @@ GEM
     sass (3.1.15)
     shotgun (0.9)
       rack (>= 1.0)
+    simplecov (0.6.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.5.3)
+    simplecov-html (0.5.3)
     sinatra (1.3.2)
       rack (~> 1.3, >= 1.3.6)
       rack-protection (~> 1.2)
@@ -158,7 +163,7 @@ GEM
       hike (~> 1.2)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
-    systemu (2.4.2)
+    systemu (2.5.0)
     thor (0.14.6)
     tilt (1.3.3)
     uuid (2.3.5)

hobson.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "open4"
   s.add_runtime_dependency "uuid"
   s.add_runtime_dependency "thor", "~> 0.14.6"
+  s.add_runtime_dependency "simplecov"
 
   s.add_runtime_dependency "rack", "~> 1.3.5"
   s.add_runtime_dependency "vegas"

lib/hobson/project/test_run/job/actions.rb

@@ -1,12 +1,45 @@
 require 'timeout'
+require 'simplecov'
+
 
 class Hobson::Project::TestRun::Job
 
+  FINISH_TEST_RUN_LOCK = 'finish_test_run_lock'
+
   def enqueue!
     Hobson.resque.enqueue(Hobson::Project::TestRun::Runner, test_run.project.name, test_run.id, index)
     enqueued!
   end
 
+  def lock_can_finish_test_run
+    # test_run.redis
+    result = false
+    until result == true
+      result = test_run.project.redis.setnx(FINISH_TEST_RUN_LOCK, 'lock')
+      sleep 0.1
+      #TODO: check for lock expiration
+    end
+  end
+
+  def release_can_finish_test_run
+    test_run.project.redis.del FINISH_TEST_RUN_LOCK
+  end
+
+  def can_finish_test_run?
+    #make sure only one job is in here at any time:
+    lock_can_finish_test_run
+    begin
+      # mark this job as ready for wrap up
+      ready_to_finish_run!
+
+      #This reload seems unnecessary, but the status is cached, and not cleared (except locally) on write.
+      test_run.redis_hash.reload!
+      test_run.jobs.all?(&:ready_to_finish_run?)
+    ensure
+      release_can_finish_test_run
+    end
+  end
+
   def run_tests!
     return if test_run.aborted?
 
@@ -56,6 +89,12 @@ def run_tests!
     tearing_down!
     eval_hook :teardown
 
+    if can_finish_test_run?
+      finishing_test_run!
+      eval_hook :finish_test_run
+      save_test_run_artifacts!
+    end
+
   rescue Object => e
     logger.info %(Exception:\n#{e}\n#{e.backtrace.join("\n")})
     self['exception'] = e.to_s
@@ -65,6 +104,7 @@ def run_tests!
     raise # raise so resque shows this as a failed job and you can retry it
   ensure
     complete!
+
     begin
       save_log_files!
     rescue Exception => e
@@ -77,6 +117,12 @@ def save_log_files!
     save_artifact(Hobson.temp_logfile.tap(&:flush).path, :name => 'test_run.log') if Hobson.temp_logfile.present?
   end
 
+  def save_test_run_artifacts!
+    artifacts_dir = workspace.root.join('artifacts')
+    logger.debug "ARTIE DIR: #{artifacts_dir.to_s}"
+    artifacts_dir.children.each{|path| save_artifact path, :content_type => `file -Ib #{path}`.gsub(/\n/,"") } if File.directory?(artifacts_dir)
+  end
+
   def abort?
     return false if !running? || !test_run.aborted?
   end

lib/hobson/project/test_run/job/status.rb

@@ -9,7 +9,9 @@ class Hobson::Project::TestRun::Job
     :running_tests,
     :saving_artifacts,
     :tearing_down,
-    :complete
+    :complete,
+    :ready_to_finish_run,
+    :finishing_test_run
 
   def abort!
     aborting! unless complete?
@@ -28,14 +30,16 @@ def complete?
   end
 
   def status
-    errored?           ? 'errored'           :
-    complete?          ? 'complete'          :
-    tearing_down?      ? 'tearing down'      :
-    saving_artifacts?  ? 'saving artifacts'  :
-    running_tests?     ? 'running tests'     :
-    preparing?         ? 'preparing'         :
-    checking_out_code? ? 'checking out code' :
-    enqueued?          ? 'waiting to be run' :
+    errored?              ? 'errored'                      :
+    complete?             ? 'complete'                     :
+    tearing_down?         ? 'tearing down'                 :
+    saving_artifacts?     ? 'saving artifacts'             :
+    running_tests?        ? 'running tests'                :
+    preparing?            ? 'preparing'                    :
+    checking_out_code?    ? 'checking out code'            :
+    enqueued?             ? 'waiting to be run'            :
+    ready_to_finish_run?  ? 'tests completed for this job' :
+    finishing_test_run?   ? 'finishing test run'           :
     'waiting...'
   end