Merge 273a400 into 4e8685d

Gemfile

@@ -4,10 +4,10 @@ gem "patir", "~>0.8", :require => false
 gem "highline","~>1.7", :require => false
 
 group :development do
-  gem 'coveralls', '>= 0.8.21', :require => false
-	gem "hoe", "~> 3.16.1", :require => false
-	gem 'rdoc','~>5.0.0', :require => false
-	gem "mocha","~>1.2.1", :require => false
-  gem "pry", :require => false
-  gem 'test-unit','~>3.2.3',:require => false
-end
+  gem 'coveralls', '~>0.8.23', :require => false
+  gem "hoe", "~>3.22.1", :require => false
+  gem 'rdoc','~>5.1.0', :require => false
+  gem "mocha","~>1.11.2", :require => false
+  gem "pry",'~>0.13.1',:require => false
+  gem 'test-unit','~>3.3.6',:require => false
+end

Gemfile.lock

@@ -1,7 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    coderay (1.1.2)
+    coderay (1.1.3)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
@@ -10,21 +10,19 @@ GEM
       tins (~> 1.6)
     docile (1.3.2)
     highline (1.7.8)
-    hoe (3.16.2)
-      rake (>= 0.8, < 13.0)
+    hoe (3.22.1)
+      rake (>= 0.8, < 15.0)
     json (2.3.0)
-    metaclass (0.0.4)
-    method_source (0.8.2)
-    mocha (1.2.1)
-      metaclass (~> 0.0.1)
+    method_source (1.0.0)
+    mocha (1.11.2)
     patir (0.9.0)
       systemu (~> 2.6)
-    power_assert (1.1.0)
-    pry (0.11.0)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-    rake (12.3.3)
-    rdoc (5.0.0)
+    power_assert (1.2.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.0.1)
+    rdoc (5.1.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -34,7 +32,7 @@ GEM
     systemu (2.6.5)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
-    test-unit (3.2.6)
+    test-unit (3.3.6)
       power_assert
     thor (1.0.1)
     tins (1.24.1)
@@ -44,14 +42,14 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  coveralls (>= 0.8.21)
+  coveralls (~> 0.8.23)
   highline (~> 1.7)
-  hoe (~> 3.16.1)
-  mocha (~> 1.2.1)
+  hoe (~> 3.22.1)
+  mocha (~> 1.11.2)
   patir (~> 0.8)
-  pry
-  rdoc (~> 5.0.0)
-  test-unit (~> 3.2.3)
+  pry (~> 0.13.1)
+  rdoc (~> 5.1.0)
+  test-unit (~> 3.3.6)
 
 BUNDLED WITH
    1.17.3

Rakefile

@@ -9,7 +9,7 @@ Hoe.spec "rutema" do |prj|
   license "MIT"
   prj.version = Rutema::Version::STRING
   prj.summary='rutema is a test execution and management framework for heterogeneous testing environments'
-  prj.urls=["http://github.com/damphyr/rutema"]
+  prj.urls={ "home" => "http://github.com/damphyr/rutema" }
   prj.description= "rutema is a test execution tool and a framework for organizing and managing test execution across different tools.\nIt enables the combination of different test tools while it takes care of logging, reporting, archiving of results and formalizes execution of automated and manual tests.\nIt's purpose is to make testing in heterogeneous environments easier."
   prj.local_rdoc_dir='doc/rdoc'
   prj.readme_file="README.md"

lib/rutema/core/configuration.rb

@@ -189,6 +189,7 @@ def import filename
           raise ConfigurationException, "Import error: Can't find #{fnm}"
         end
       end
+
       private
       def load_configuration filename
         begin 

lib/rutema/core/engine.rb

@@ -39,11 +39,9 @@ def run test_identifier=nil
         @dispatcher.exit
         raise RutemaError,"No tests to run!"
       else
-        @runner.setup=setup
-        @runner.teardown=teardown
         #running - at this point we've done any and all checks and we're stepping on the gas
         message("running")
-        run_scenarios(tests,suite_setup,suite_teardown)
+        run_scenarios(tests,suite_setup,suite_teardown,setup,teardown)
       end
       message("end")
       @dispatcher.exit
@@ -101,27 +99,30 @@ def parse_specials configuration
       end
       return suite_setup,suite_teardown,setup,teardown
     end
-    def run_scenarios specs,suite_setup,suite_teardown
+    def run_scenarios specs,suite_setup,suite_teardown,setup,teardown
       if specs.empty?
         error(nil,"No tests to run")
       else
-        if suite_setup
-          if run_test(suite_setup)==:success
-            specs.each{|s| run_test(s)}
-          else
-            error(nil,"Suite setup test failed")
-          end
-        else
+        @runner.setup=nil
+        @runner.teardown=nil
+ 
+        if !suite_setup || (run_test(suite_setup, true)==:success)
+          @runner.setup=setup
+          @runner.teardown=teardown
           specs.each{|spec| run_test(spec)}
+        else
+          error(nil,"Suite setup test failed")          
         end
         if suite_teardown
-          run_test(suite_teardown)
+          @runner.setup=nil
+          @runner.teardown=nil
+          run_test(suite_teardown, true)
         end
       end
     end
-    def run_test specification
+    def run_test specification, is_special = false
       if specification.scenario
-        status=@runner.run(specification)["status"]
+        status=@runner.run(specification, is_special)["status"]
       else
         status=:not_executed
         message(:test=>specification.name,:text=>"No scenario", :status=>status)
@@ -135,16 +136,18 @@ def instantiate_class definition,configuration
       end
       return nil
     end
+    
     def is_spec_included? test_identifier
       full_path=File.expand_path(test_identifier)
-      return @configuration.tests.include?(full_path) || is_special?(test_identifier) 
+      return @configuration.tests.include?(full_path) || is_special?(test_identifier)
     end
+
     def is_special? test_identifier
       full_path=File.expand_path(test_identifier)
       return full_path==@configuration.suite_setup ||
       full_path==@configuration.suite_teardown ||
       full_path==@configuration.setup ||
-      full_path==@configuration.teardown 
+      full_path==@configuration.teardown  
     end
   end
   #The Rutema::Dispatcher functions as a demultiplexer between Rutema::Engine and the various reporters.
@@ -224,4 +227,4 @@ def dispatch
       end
     end
   end
-end
+end

lib/rutema/core/framework.rb

@@ -1,4 +1,8 @@
+#  Copyright (c) 2007-2020 Vassilis Rizopoulos. All rights reserved.
 module Rutema
+
+  STATUS_CODES=[:started,:skipped,:success,:warning,:error]
+
   #Represents the data beeing shunted between the components in lieu of logging.
   #
   #This is the primary type passed to the event reporters
@@ -36,28 +40,32 @@ def to_s
   #If there is an engine error (e.g. when parsing) you will get an ErrorMessage, if it is a test error
   #you will get a RunnerMessage with :error in the status.
   class RunnerMessage<Message
-    attr_accessor :duration,:status,:number,:out,:err
+    attr_accessor :duration,:status,:number,:out,:err,:is_special
     def initialize params
       super(params)
       @duration=params.fetch("duration",0)
       @status=params.fetch("status",:none)
       @number=params.fetch("number",1)
       @out=params.fetch("out","")
       @err=params.fetch("err","")
+      @backtrace=params.fetch("backtrace","")
+      @is_special=params.fetch("is_special","")
     end
 
     def to_s
       msg="#{@test}:"
+      msg<<" #{@timestamp.strftime("%H:%M:%S")} :"
       msg<<"#{@text}." unless @text.empty?
       outpt=output()
-      msg<<" Output:\n#{outpt}" unless outpt.empty? || @status!=:error
+      msg<<" Output" + (outpt.empty? ? "." : ":\n#{outpt}") # unless outpt.empty? || @status!=:error
       return msg
     end
 
     def output
       msg=""
       msg<<"#{@out}\n" unless @out.empty?
       msg<<@err unless @err.empty?
+      msg<<"\n" + (@backtrace.kind_of?(Array) ? @backtrace.join("\n") : @backtrace) unless @backtrace.empty? 
       return msg.chomp
     end
   end
@@ -68,20 +76,21 @@ def output
   #and accumulates the duration reported by all messages in it's collection.
   class ReportState
     attr_accessor :steps
-    attr_reader :test,:timestamp,:duration,:status
+    attr_reader :test,:timestamp,:duration,:status,:is_special
     
     def initialize message
       @test=message.test
       @timestamp=message.timestamp
       @duration=message.duration
       @status=message.status
       @steps=[message]
+      @is_special=message.is_special
     end
 
     def <<(message)
       @steps<<message
       @duration+=message.duration
-      @status=message.status
+      @status=message.status unless message.status.nil? || (!@status.nil? && STATUS_CODES.find_index(message.status) < STATUS_CODES.find_index(@status))
     end
   end
 
@@ -94,7 +103,7 @@ def error identifier,message
     def message message
       case message
       when String
-        Message.new(:text=>message,:timestamp=>Time.now)
+        @queue.push(Message.new(:text=>message,:timestamp=>Time.now))
       when Hash
         hm=Message.new(message)
         hm=RunnerMessage.new(message) if message[:test] && message["status"]

lib/rutema/core/objectmodel.rb

@@ -122,6 +122,11 @@ def steps= array_of_steps
   #
   #ignore - set to true if the step's success or failure is to be ignored. It essentially means that the step is always considered succesfull
   #
+  #continue - set to true if the step's success or failure is to be recognized, but following steps in the same scenario are to be carried out regardless
+  #           It indicates test failure, but ensures any further state is carried out as needed
+  #
+  # skip_on_error - if the test case has been marked a failure, skip steps marked with this attribute
+  #
   #number - this is set when the step is assigned to a Scenario and is the sequence number
   #
   #cmd - the command associated with this step. This should quack like Patir::Command.
@@ -152,6 +157,10 @@ def initialize txt="",cmd=nil
       @attributes=Hash.new
       #ignore is off by default
       @attributes[:ignore]=false
+      #continue is off by default
+      @attributes[:continue]=false
+      #skip_on_error is off by default
+      @attributes[:skip_on_error]=false
       #assign
       @attributes[:cmd]=cmd if cmd
       @attributes[:text]=txt
@@ -170,10 +179,22 @@ def error
       return "no command associated" unless @attributes[:cmd]
       return @attributes[:cmd].error
     end
+    def backtrace
+      return "no backtrace associated" unless @attributes[:cmd]
+      return @attributes[:cmd].backtrace
+    end
+    def skip_on_error?
+      return false unless @attributes[:skip_on_error]
+      return @attributes[:skip_on_error]
+    end
     def ignore?
       return false unless @attributes[:ignore]
       return @attributes[:ignore]
     end
+    def continue?
+      return false unless @attributes[:continue]
+      return @attributes[:continue]
+    end    
     def exec_time
       return 0 unless @attributes[:cmd]
       return @attributes[:cmd].exec_time

lib/rutema/core/reporter.rb

@@ -23,16 +23,13 @@ def initialize configuration,dispatcher
       def run!
         @thread=Thread.new do
           while true do
-            if  @queue.size>0
-              data=@queue.pop
-              begin
-                update(data) if data
-              rescue
-                puts "#{self.class} failed with #{$!.message}"
-                raise
-              end
+            data=@queue.pop
+            begin
+              update(data) if data
+            rescue
+              puts "#{self.class} failed with #{$!.message}"
+              raise
             end
-            sleep 0.1
           end
         end
       end
@@ -94,8 +91,10 @@ def update message
           when RunnerMessage
             if message.status == :error
               puts "FATAL|#{message.to_s}"
+            elsif message.status == :warning
+              puts "WARNING|#{message.to_s}"
             else
-              puts message.to_s if @mode=="verbose"
+              puts "#{message.to_s} #{message.status}." if @mode=="verbose"
             end
           when ErrorMessage
             puts message.to_s 
@@ -116,7 +115,8 @@ def report specs,states,errors
         states.each{|k,v| failures<<v.test if v.status==:error}
 
         unless @silent
-          puts "#{errors.size} errors. #{states.size} test cases executed. #{failures.size} failed"
+          count_tests_run = states.select { |name, state| !state.is_special }.count
+          puts "#{errors.size} errors. #{count_tests_run} test cases executed. #{failures.size} failed"
           unless failures.empty?
             puts "Failures:"
             puts specs.map{|spec| "  #{spec.name} - #{spec.filename}" if failures.include?(spec.name)}.compact.join("\n")