print profile summary and test summary

lib/inspec/profile.rb

@@ -142,8 +142,12 @@ def to_s
       "Inspec::Profile<#{name}>"
     end
 
-    def info
-      res = params.dup
+    # return info using uncached params
+    def info!
+      info(load_params.dup)
+    end
+
+    def info(res = params.dup)
       # add information about the controls
       controls = res[:controls].map do |id, rule|
         next if id.to_s.empty?

lib/inspec/profile_context.rb

@@ -17,7 +17,8 @@ def self.for_profile(profile, backend, attributes)
                                    'attributes' => attributes })
     end
 
-    attr_reader :attributes, :rules, :profile_id, :resource_registry
+    attr_reader :attributes, :profile_id, :resource_registry
+    attr_accessor :rules
     def initialize(profile_id, backend, conf)
       if backend.nil?
         fail 'ProfileContext is initiated with a backend == nil. ' \
@@ -132,7 +133,12 @@ def unregister_rule(id)
 
     def register_rule(r)
       # get the full ID
-      r.instance_variable_set(:@__file, current_load[:file])
+      file = if @current_load.nil?
+               'unknown'
+             else
+               @current_load[:file] || 'unknown'
+             end
+      r.instance_variable_set(:@__file, file)
       r.instance_variable_set(:@__group_title, current_load[:title])
 
       # add the rule to the registry

lib/inspec/rspec_json_formatter.rb

@@ -36,11 +36,8 @@ class InspecRspecMiniJson < RSpec::Core::Formatters::JsonFormatter
   # Called after stop has been called and the run is complete.
   def dump_summary(summary)
     @output_hash[:version] = Inspec::VERSION
-    @output_hash[:summary] = {
+    @output_hash[:statistics] = {
       duration: summary.duration,
-      example_count: summary.example_count,
-      failure_count: summary.failure_count,
-      skip_count: summary.pending_count,
     }
   end
 
@@ -86,7 +83,7 @@ def format_example(example)
   end
 end
 
-class InspecRspecJson < InspecRspecMiniJson
+class InspecRspecJson < InspecRspecMiniJson # rubocop:disable Metrics/ClassLength
   RSpec::Core::Formatters.register self, :start, :stop, :dump_summary
   attr_writer :backend
 
@@ -108,7 +105,7 @@ def add_profile(profile)
   def start(_notification)
     # Note that the default profile may have no name - therefore
     # the hash may have a valid nil => entry.
-    @profiles_info ||= Hash[@profiles.map { |x| profile_info(x) }]
+    @profiles_info = Hash[@profiles.map { |x| profile_info(x) }]
   end
 
   def dump_one_example(example, control)
@@ -133,19 +130,27 @@ def stop(notification)
     @output_hash[:other_checks] = missing
   end
 
-  def dump_summary(summary)
-    super(summary)
-    total = 0
+  def controls_summary
     failed = 0
     skipped = 0
     passed = 0
+    critical = 0
+    major = 0
+    minor = 0
 
     @profiles_info.each do |_name, profile|
-      total += profile[:controls].length
       profile[:controls].each do |_control_name, control|
+        next if control[:id].start_with? '(generated from '
         next unless control[:results]
         if control[:results].any? { |r| r[:status] == 'failed' }
           failed += 1
+          if control[:impact] >= 0.7
+            critical += 1
+          elsif control[:impact] >= 0.4
+            major += 1
+          else
+            minor += 1
+          end
         elsif control[:results].any? { |r| r[:status] == 'skipped' }
           skipped += 1
         else
@@ -154,13 +159,45 @@ def dump_summary(summary)
       end
     end
 
-    # TODO: provide this information in the output
+    total = failed + passed + skipped
+
+    { 'total' => total,
+      'failed' => {
+        'total' => failed,
+        'critical' => critical,
+        'major' => major,
+        'minor' => minor,
+      },
+      'skipped' => skipped,
+      'passed' => passed }
+  end
+
+  def tests_summary
+    total = 0
+    failed = 0
+    skipped = 0
+    passed = 0
+
+    @anonymous_tests.each do |control|
+      next unless control[:results]
+      control[:results].each do |result|
+        if result[:status] == 'failed'
+          failed +=1
+        elsif result[:status] == 'skipped'
+          skipped +=1
+        else
+          passed +=1
+        end
+      end
+    end
+
+    { 'total' => total, 'failed' => failed, 'skipped' => skipped, 'passed' => passed }
   end
 
   private
 
   def profile_info(profile)
-    info = profile.info.dup
+    info = profile.info!.dup
     [info[:name], info]
   end
 
@@ -249,7 +286,7 @@ def initialize(*args)
     super(*args)
   end
 
-  def close(_notification)
+  def close(_notification) # rubocop:disable Metrics/AbcSize
     flush_current_control
     output.puts('') unless @current_control.nil?
     print_tests
@@ -266,13 +303,26 @@ def close(_notification)
       output.puts('')
     end
 
-    res = @output_hash[:summary]
-    passed = res[:example_count] - res[:failure_count] - res[:skip_count]
+    controls_res = controls_summary
+    tests_res = tests_summary
+
+    s = format('Profile Summary: %s%d successful%s, %s%d failures%s, %s%d skipped%s',
+               COLORS['passed'], controls_res['passed'], COLORS['reset'],
+               COLORS['failed'], controls_res['failed']['total'], COLORS['reset'],
+               COLORS['skipped'], controls_res['skipped'], COLORS['reset'])
+    output.puts(s) if controls_res['total'] > 0
+
+    s = format('Test Summary: %s%d successful%s, %s%d failures%s, %s%d skipped%s',
+               COLORS['passed'], tests_res['passed'], COLORS['reset'],
+               COLORS['failed'], tests_res['failed'], COLORS['reset'],
+               COLORS['skipped'], tests_res['skipped'], COLORS['reset'])
+    output.puts(s) if !@anonymous_tests.empty?
+
     s = format('Summary: %s%d successful%s, %s%d failures%s, %s%d skipped%s',
-               COLORS['passed'], passed, COLORS['reset'],
-               COLORS['failed'], res[:failure_count], COLORS['reset'],
-               COLORS['skipped'], res[:skip_count], COLORS['reset'])
-    output.puts(s)
+               COLORS['passed'], 0, COLORS['reset'],
+               COLORS['failed'], 0, COLORS['reset'],
+               COLORS['skipped'], 0, COLORS['reset'])
+    output.puts(s) if @current_control.nil? || (@anonymous_tests.empty? && controls_res['total'] == 0)
   end
 
   private

lib/inspec/runner.rb

@@ -34,7 +34,7 @@ class Runner # rubocop:disable Metrics/ClassLength
     def_delegator :@test_collector, :report
     def_delegator :@test_collector, :reset
 
-    attr_reader :backend, :rules, :attributes
+    attr_reader :backend, :rules, :attributes, :target_profiles
     def initialize(conf = {})
       @rules = []
       @conf = conf.dup

lib/inspec/shell.rb

@@ -18,7 +18,13 @@ def start
       # Create an in-memory empty runner so that we can add tests to it later.
       # This context lasts for the duration of this "start" method call/pry
       # session.
-      @ctx = @runner.create_context
+
+      # require fetchers/mock here to ensure it is only loaded when we are actually using the shell
+      # loading it at the top of the file adds it to the fetcher registry which affects fetcher resolution
+      require 'fetchers/mock'
+      @runner.add_target({ 'inspec.yml' => 'name: inspec-shell' })
+      @our_profile = @runner.target_profiles.first
+      @ctx = @our_profile.runner_context
       configure_pry
 
       # This will hold a single evaluation binding context as opened within
@@ -51,26 +57,20 @@ def configure_pry # rubocop:disable Metrics/AbcSize
 
       # Track the rules currently registered and what their merge count is.
       Pry.hooks.add_hook(:before_eval, 'inspec_before_eval') do
-        @current_eval_rules = @ctx.rules.each_with_object({}) do |(rule_id, rule), h|
-          h[rule_id] = Inspec::Rule.merge_count(rule)
-        end
+        @ctx.rules = {}
         @runner.reset
       end
 
       # After pry has evaluated a commanding within the binding context of a
       # test file, register all the rules it discovered.
       Pry.hooks.add_hook(:after_eval, 'inspec_after_eval') do
-        @current_eval_new_tests =
-          @runner.register_rules(@ctx) do |rule_id, rule|
-            @current_eval_rules[rule_id] != Inspec::Rule.merge_count(rule)
-          end
-        @runner.run if @current_eval_new_tests
+        @runner.run if !@ctx.rules.empty?
       end
 
       # Don't print out control class inspection when the user uses DSL methods.
       # Instead produce a result of evaluating their control.
       Pry.config.print = proc do |_output_, value, pry|
-        next if @current_eval_new_tests
+        next if !@ctx.rules.empty?
         pry.pager.open do |pager|
           pager.print pry.config.output_prefix
           Pry::ColorPrinter.pp(value, pager, Pry::Terminal.width! - 1)

test/functional/inspec_exec_test.rb

@@ -18,7 +18,8 @@
 \e[37m  ○  gordon-1.0: Verify the version number of Gordon (1 skipped)\e[0m
 \e[37m     ○  Can't find file \"/tmp/gordon/config.yaml\"\e[0m
 "
-    stdout.must_include "\nSummary: \e[32m4 successful\e[0m, \e[31m0 failures\e[0m, \e[37m1 skipped\e[0m\n"
+    stdout.must_include "\nProfile Summary: \e[32m2 successful\e[0m, \e[31m0 failures\e[0m, \e[37m1 skipped\e[0m"
+    stdout.must_include "\nTest Summary: \e[32m1 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m\n"
   end
 
   it 'executes a minimum metadata-only profile' do
@@ -78,14 +79,15 @@
     out = inspec('exec ' + example_profile + ' --controls tmp-1.0')
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
-    out.stdout.must_include "\nSummary: \e[32m1 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m\n"
+    out.stdout.must_include "\nProfile Summary: \e[32m1 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m\n"
   end
 
   it 'can execute a simple file with the default formatter' do
     out = inspec('exec ' + example_control)
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
-    out.stdout.must_include "\nSummary: \e[32m2 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m\n"
+    out.stdout.must_include "\nProfile Summary: \e[32m1 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m\n"
+    out.stdout.must_include "\nTest Summary: \e[32m1 successful\e[0m, \e[31m0 failures\e[0m"
   end
 
   describe 'with a profile that is not supported on this OS/platform' do
@@ -137,7 +139,8 @@
   File /tmp
 \e[32m     \xE2\x9C\x94  should be directory\e[0m
 
-Summary: \e[32m2 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m
+Profile Summary: \e[32m1 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m
+Test Summary: \e[32m1 successful\e[0m, \e[31m0 failures\e[0m, \e[37m0 skipped\e[0m
 "
     end
   end

test/functional/inspec_shell_test.rb

@@ -63,9 +63,7 @@ def do_shell_c(code, exit_status, json = false, stderr = '')
       j.keys.must_include 'version'
       j.keys.must_include 'profiles'
       j.keys.must_include 'other_checks'
-      j.keys.must_include 'summary'
-      j['summary']['example_count'].must_equal 1
-      j['summary']['failure_count'].must_equal 0
+      j.keys.must_include 'statistics'
     end
 
     it 'runs anonymous tests that succeed' do
@@ -80,9 +78,7 @@ def do_shell_c(code, exit_status, json = false, stderr = '')
       j.keys.must_include 'version'
       j.keys.must_include 'profiles'
       j.keys.must_include 'other_checks'
-      j.keys.must_include 'summary'
-      j['summary']['example_count'].must_equal 1
-      j['summary']['failure_count'].must_equal 1
+      j.keys.must_include 'statistics'
     end
 
     it 'runs anonymous tests that fail' do
@@ -97,9 +93,7 @@ def do_shell_c(code, exit_status, json = false, stderr = '')
       j.keys.must_include 'version'
       j.keys.must_include 'profiles'
       j.keys.must_include 'other_checks'
-      j.keys.must_include 'summary'
-      j['summary']['example_count'].must_equal 1
-      j['summary']['failure_count'].must_equal 0
+      j.keys.must_include 'statistics'
     end
 
     it 'runs controls with tests' do
@@ -114,14 +108,12 @@ def do_shell_c(code, exit_status, json = false, stderr = '')
       j.keys.must_include 'version'
       j.keys.must_include 'profiles'
       j.keys.must_include 'other_checks'
-      j.keys.must_include 'summary'
-      j['summary']['example_count'].must_equal 2
-      j['summary']['failure_count'].must_equal 1
+      j.keys.must_include 'statistics'
     end
 
     it 'runs controls with multiple tests' do
       out = do_shell_c("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end; describe file(\"foo/bar/baz\") do it { should exist } end end", 1)
-      out.stdout.must_include '1 successful'
+      out.stdout.must_include '0 successful'
       out.stdout.must_include '1 failures'
     end
   end
@@ -178,7 +170,7 @@ def do_shell(code, exit_status = 0, stderr = '')
 
     it 'runs controls with multiple tests' do
       out = do_shell("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end; describe file(\"foo/bar/baz\") do it { should exist } end end")
-      out.stdout.must_include '1 successful'
+      out.stdout.must_include '0 successful'
       out.stdout.must_include '1 failures'
     end