Finishing unit and functional test coverage for audit runner

Gemfile

@@ -2,6 +2,8 @@ source "https://rubygems.org"
 gemspec :name => "chef"
 
 gem "activesupport", "< 4.0.0", :group => :compat_testing, :platform => "ruby"
+# TODO after serverspec stops including their top level DSL we can remove this
+gem "serverspec", :git => "https://github.com/tyler-ball/serverspec/"
 
 group(:docgen) do
   gem "yard"

lib/chef/audit/runner.rb

@@ -74,9 +74,6 @@ def setup
       # prevents Specinfra and Serverspec from modifying the RSpec configuration
       # used by our spec tests.
       def require_deps
-        # TODO: We need to figure out a way to give audits its own configuration
-        # object. This involves finding a way to load these files w/o them adding
-        # to the configuration object used by our spec tests.
         require 'rspec'
         require 'rspec/its'
         require 'specinfra'
@@ -143,7 +140,7 @@ def configure_specinfra
         # TODO: We may need to be clever and adjust this based on operating
         # system, or make it configurable. E.g., there is a PowerShell backend,
         # as well as an SSH backend.
-        Specinfra.configuration.backend = :exec
+        Specinfra.configuration.backend = :exec if Specinfra.configuration.backend != :exec
       end
 
       # Iterates through the controls registered to this run_context, builds an

spec/functional/audit/runner_spec.rb

@@ -1,10 +1,25 @@
+#
+# Author:: Tyler Ball (<tball@chef.io>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 require 'spec_helper'
 require 'spec/support/audit_helper'
 require 'chef/audit/runner'
-require 'chef/audit/audit_event_proxy'
-require 'chef/audit/rspec_formatter'
-require 'chef/run_context'
-require 'pry'
+require 'rspec/support/spec/in_sub_process'
 
 ##
 # This functional test ensures that our runner can be setup to not interfere with existing RSpec
@@ -13,68 +28,80 @@
 
 describe Chef::Audit::Runner do
 
-  let(:events) { double("events") }
-  let(:run_context) { instance_double(Chef::RunContext) }
-  let(:runner) { Chef::Audit::Runner.new(run_context) }
-
-  # This is the first thing that gets called, and determines how the examples are ran
-  around(:each) do |ex|
-    Sandboxing.sandboxed { ex.run }
-  end
-
-  describe "#configure_rspec" do
-
-    it "adds the necessary formatters" do
-      # We don't expect the events to receive any calls because the AuditEventProxy that was registered from `runner.run`
-      # only existed in the Configuration object that got removed by the sandboxing
-      #expect(events).to receive(:control_example_success)
-
-      expect(RSpec.configuration.formatters.size).to eq(0)
-      expect(run_context).to receive(:events).and_return(events)
-      expect(Chef::Audit::AuditEventProxy).to receive(:events=)
+  # The functional tests must be run in a sub_process.  Including Serverspec includes the Serverspec DSL - this
+  # conflicts with our `package` DSL (among others) when we try to test `package` inside an RSpec example.
+  # Our DSL leverages `method_missing` while the Serverspec DSL defines a method on the RSpec::Core::ExampleGroup.
+  # The defined method wins our and returns before our `method_missing` DSL can be called.
+  #
+  # Running in a sub_process means the serverspec libraries will only be included in a forked process, not the main one.
+  include RSpec::Support::InSubProcess
 
-      runner.send(:add_formatters)
+    let(:events) { double("events").as_null_object }
+    let(:runner) { Chef::Audit::Runner.new(run_context) }
+    let(:stdout) { StringIO.new }
 
-      expect(RSpec.configuration.formatters.size).to eq(2)
-      expect(RSpec.configuration.formatters[0]).to be_instance_of(Chef::Audit::AuditEventProxy)
-      expect(RSpec.configuration.formatters[1]).to be_instance_of(Chef::Audit::RspecFormatter)
+    around(:each) do |ex|
+      Sandboxing.sandboxed { ex.run }
+    end
 
+    before do
+      Chef::Config[:log_location] = stdout
     end
 
-  end
+    # When running these, because we are not mocking out any of the formatters we expect to get dual output on the
+    # command line
+    describe "#run" do
 
-  # When running these, because we are not mocking out any of the formatters we expect to get dual output on the
-  # command line
-  describe "#run" do
+      let(:audits) { {} }
+      let(:run_context) { instance_double(Chef::RunContext, :events => events, :audits => audits) }
+      let(:controls_name) { "controls_name" }
 
-    before do
-      expect(run_context).to receive(:events).and_return(events)
-    end
+      it "Correctly runs an empty controls block" do
+        in_sub_process do
+          runner.run
+        end
+      end
 
-    it "Correctly runs an empty controls block" do
-      expect(run_context).to receive(:audits).and_return({})
-      runner.run
-    end
+      context "there is a single successful control" do
+        let(:audits) do
+          should_pass = lambda do
+            it "should pass" do
+              expect(2 - 2).to eq(0)
+            end
+          end
+          { controls_name => Struct.new(:args, :block).new([controls_name], should_pass)}
+        end
 
-    it "Correctly runs a single successful control" do
-      should_pass = lambda do
-        it "should pass" do
-          expect(2 - 2).to eq(0)
+        it "correctly runs" do
+          in_sub_process do
+            runner.run
+
+            expect(stdout.string).to match(/1 example, 0 failures/)
+          end
         end
       end
 
-      expect(run_context).to receive(:audits).and_return({
-        "should pass" => {:args => [], :block => should_pass}
-      })
+      context "there is a single failing control" do
+        let(:audits) do
+          should_fail = lambda do
+            it "should fail" do
+              expect(2 - 1).to eq(0)
+            end
+          end
+          { controls_name => Struct.new(:args, :block).new([controls_name], should_fail)}
+        end
 
-      # TODO capture the output and verify it
-      runner.run
-    end
+        it "correctly runs" do
+          in_sub_process do
+            runner.run
 
-    it "Correctly runs a single failing control", :pending do
+            expect(stdout.string).to match(/Failure\/Error: expect\(2 - 1\)\.to eq\(0\)/)
+            expect(stdout.string).to match(/1 example, 1 failure/)
+            expect(stdout.string).to match(/# controls_name should fail/)
+          end
+        end
+      end
 
     end
 
-  end
-
 end

spec/support/audit_helper.rb

@@ -8,7 +8,8 @@
 
 # rspec-core did not include a license on Github
 
-# This is necessary, otherwise
+# Adding these as writers is necessary, otherwise we cannot set the new configuration.
+# Only want to do this in the specs.
 class << RSpec
   attr_writer :configuration, :world
 end

spec/unit/audit/runner_spec.rb

@@ -0,0 +1,124 @@
+#
+# Author:: Tyler Ball (<tball@chef.io>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'spec/support/audit_helper'
+require 'chef/audit/runner'
+
+describe Chef::Audit::Runner do
+
+  let(:events) { double("events") }
+  let(:run_context) { instance_double(Chef::RunContext, :events => events) }
+  let(:runner) { Chef::Audit::Runner.new(run_context) }
+
+  around(:each) do |ex|
+    Sandboxing.sandboxed { ex.run }
+  end
+
+  describe "#initialize" do
+    it "correctly sets the run_context during initialization" do
+      expect(runner.instance_variable_get(:@run_context)).to eq(run_context)
+    end
+  end
+
+  context "during #run" do
+
+    describe "#setup" do
+      let(:log_location) { File.join(Dir.tmpdir, 'audit_log') }
+      let(:color) { false }
+
+      before do
+        Chef::Config[:log_location] = log_location
+        Chef::Config[:color] = color
+      end
+
+      it "sets all the config values" do
+        runner.send(:setup)
+
+        expect(RSpec.configuration.output_stream).to eq(log_location)
+        expect(RSpec.configuration.error_stream).to eq(log_location)
+
+        expect(RSpec.configuration.formatters.size).to eq(2)
+        expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::AuditEventProxy))
+        expect(RSpec.configuration.formatters).to include(instance_of(Chef::Audit::RspecFormatter))
+        expect(Chef::Audit::AuditEventProxy.class_variable_get(:@@events)).to eq(run_context.events)
+
+        expect(RSpec.configuration.expectation_frameworks).to eq([RSpec::Matchers])
+        expect(RSpec::Matchers.configuration.syntax).to eq([:expect])
+
+        expect(RSpec.configuration.color).to eq(color)
+        expect(RSpec.configuration.expose_dsl_globally?).to eq(false)
+
+        expect(Specinfra.configuration.backend).to eq(:exec)
+      end
+    end
+
+    describe "#register_controls" do
+      let(:audits) { [] }
+      let(:run_context) { instance_double(Chef::RunContext, :audits => audits) }
+
+      it "adds the control group aliases" do
+        runner.send(:register_controls)
+
+        expect(RSpec::Core::DSL.example_group_aliases).to include(:__controls__)
+        expect(RSpec::Core::DSL.example_group_aliases).to include(:control)
+      end
+
+      context "audits exist" do
+        let(:audits) { {"audit_name" => group} }
+        let(:group) {Struct.new(:args, :block).new(["group_name"], nil)}
+
+        it "sends the audits to the world" do
+          runner.send(:register_controls)
+
+          expect(RSpec.world.example_groups.size).to eq(1)
+          # For whatever reason, `kind_of` is not working
+          g = RSpec.world.example_groups[0]
+          expect(g.ancestors).to include(RSpec::Core::ExampleGroup)
+          expect(g.description).to eq("group_name")
+        end
+      end
+    end
+
+    describe "#do_run" do
+      let(:rspec_runner) { instance_double(RSpec::Core::Runner) }
+
+      it "executes the runner" do
+        expect(RSpec::Core::Runner).to receive(:new).with(nil).and_return(rspec_runner)
+        expect(rspec_runner).to receive(:run_specs).with([])
+
+        runner.send(:do_run)
+      end
+    end
+  end
+
+  describe "counters" do
+    it "correctly calculates failed?" do
+      expect(runner.failed?).to eq(false)
+    end
+
+    it "correctly calculates num_failed" do
+      expect(runner.num_failed).to eq(0)
+    end
+
+    it "correctly calculates num_total" do
+      expect(runner.num_total).to eq(0)
+    end
+  end
+
+end