Refactor acceptance tests

CONTRIBUTING.md

@@ -1,25 +1,47 @@
 Testing
 =======
 
-Tests on the puppet-puppet module can be run via `rake`.
+Puppet-puppet is tested via beaker and rspec-puppet. Rspec-puppet tests are run
+automatically for pull requests via travis-ci, or can be triggered manually
+by running `rake spec` after doing a `bundle install`.
 
-#### Beaker acceptance tests
-```
-BEAKER_destroy=no BEAKER_provision=onpass rake acceptance
+Beaker tests are run slightly differently for puppet-puppet than for other
+beaker-tested projects because the test suite should be run independently for
+each test case to prevent contamination between environments. For example,
+running the apache-passenger based puppet master test case will cause obvious
+conflicts if the nginx-unicorn puppet master is subsequently built on the same
+virtual machine. Rspec tags are used to accomplish this in the test cases.
+
+Running beaker tests for unicorn puppetmaster on the default nodeset (debian 7):
+```bash
+rspec . --tag servertype:unicorn --pattern "spec/acceptance/**/*_spec.rb"
 ```
 
-Run beaker acceptance tests on debian 7:
+Same as above, but only only destroying the VM if the build is successful, to
+help troubleshooting:
 ```
-BEAKER_set=debian-73-x64 BEAKER_destroy=onpass BEAKER_provision=no rake acceptance
+BEAKER_destroy=onpass BEAKER_provision=yes rspec . --tag servertype:unicorn --pattern "spec/acceptance/**/*_spec.rb"
 ```
-See spec/acceptance/nodesets for a list of possible node names; use the filename without .yml.
+
+Same as above, but on a different nodeset (see `spec/acceptance/nodesets` for
+options):
+```bash
+BEAKER_set=sles-11sp1-x64 BEAKER_destroy=onpass BEAKER_provision=yes rspec . --tag servertype:unicorn --pattern "spec/acceptance/**/*_spec.rb"
+```
+
+The presence of an OS/distro in the nodeset list does not imply support. The
+SLES example above is expected to fail most test cases but is included to lower
+the bar for future contributors who want to add support for additional distros.
+
+The rake command includes acceptance testing tasks, but these should not be
+used because they will run all of the acceptance tests on the same VM, which
+is expected to fail.
 
 If a beaker test fails, you can SSH into the environment if you use BEAKER_PROVISION=onpass.
 The path of the vagrantfile will be `.vagrant/beaker_vagrant_files/debian-73-x64.yml`
 if you followed the above instructions, and slightly different if you used a
 different nodeset. `cd` to that directory and `vagrant ssh` to access the VM.
 The tests that ran are in /tmp with randomly generated filenames.
-```
 
 #### rspec-puppet tests
 (note that these are run automatically by travis CI on pull requests)

spec/acceptance/agent_spec.rb

@@ -2,7 +2,7 @@
 
 describe 'agent', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do
   context 'with default parameters' do
-    it 'should run with no errors' do
+    it 'should run with no errors', :agent do
       pp = <<-EOS
       class { 'puppet::agent': }
       EOS

spec/acceptance/nodesets/default.yml

@@ -1,10 +1,10 @@
 HOSTS:
-  ubuntu-server-12042-x64:
+  debian-73-x64:
     roles:
       - master
-    platform: ubuntu-12.04-amd64
-    box : ubuntu-server-12042-x64-vbox4210-nocm
-    box_url : http://puppet-vagrant-boxes.puppetlabs.com/ubuntu-server-12042-x64-vbox4210-nocm.box
+    platform: debian-7-amd64
+    box : debian-73-x64-virtualbox-nocm
+    box_url : http://puppet-vagrant-boxes.puppetlabs.com/debian-73-x64-virtualbox-nocm.box
     hypervisor : vagrant
 CONFIG:
   log_level: debug

spec/acceptance/passenger_server_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper_acceptance'
+
+describe 'passenger server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do
+  context 'running on passenger', :servertype => 'passenger', :webserver => 'apache' do
+    it 'should run with no errors' do
+      pp = <<-EOS
+        class { 'puppet::server':
+          servertype => 'passenger',
+          ca         => true,
+          servername => $::hostname,
+        }
+      EOS
+
+      # Run it twice and test for idempotency
+      apply_manifest(pp, :catch_failures => true)
+      expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero
+    end
+    it_behaves_like 'basic working puppetmaster'
+
+    # sanity checks to ensure the passenger setup doesn't bring in other services
+    describe service('nginx') do
+      it { should_not be_enabled }
+      it { should_not be_running }
+    end
+    describe service('puppetmaster') do
+      it { should_not be_enabled }
+      it { should_not be_running }
+    end
+    describe service('puppetserver') do
+      it { should_not be_enabled }
+      it { should_not be_running }
+    end
+
+  end
+end

spec/acceptance/thin_server_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper_acceptance'
+
+describe 'thin server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do
+
+  context 'running on thin', :servertype => 'thin', :webserver => 'nginx' do
+    it 'should run with no errors' do
+      pp = <<-EOS
+        class { "puppet::server":
+          servertype   => 'thin',
+          ca           => true,
+        }
+      EOS
+
+      # Run it twice and test for idempotency
+      apply_manifest(pp, :catch_failures => true)
+      expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero
+    end
+
+    describe package('thin') do
+      it { should be_installed.by('gem') }
+    end
+
+    describe service('thin-puppetmaster') do
+      it { should be_enabled }
+      it { should be_running }
+    end
+
+    it_behaves_like "basic working puppetmaster"
+    it_behaves_like "nginx-based webserver"
+
+  end
+end

spec/acceptance/unicorn_server_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper_acceptance'
+
+describe 'server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do
+  context 'running on unicorn', :servertype => 'unicorn', :webserver => 'nginx' do
+    it 'should run with no errors' do
+      pp = <<-EOS
+        class { 'puppet::server':
+          servertype => 'unicorn',
+          ca         => true,
+        }
+      EOS
+
+      # Run it twice and test for idempotency
+      apply_manifest(pp, :catch_failures => true)
+      expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero
+    end
+
+    it_behaves_like "basic working puppetmaster"
+    it_behaves_like "nginx-based webserver"
+
+  end
+end

spec/acceptance/webrick_server_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper_acceptance'
+
+describe 'server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do
+  context 'running on webrick/standalone', :server => 'webrick', :webserver => 'builtin' do
+    it 'should run with no errors' do
+      pp = <<-EOS
+        class { 'puppet::server':
+          servertype => 'standalone',
+          ca         => true,
+        }
+      EOS
+
+      # Run it twice and test for idempotency
+      apply_manifest(pp, :catch_failures => true)
+      expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero
+    end
+
+    describe service('puppetmaster') do
+      it { should be_enabled }
+      it { should be_running }
+    end
+
+    # sanity checks to ensure the webrick setup doesn't bring in other services
+    describe service('nginx') do
+      it { should_not be_enabled }
+      it { should_not be_running }
+    end
+    describe service('apache2') do
+      it { should_not be_enabled }
+      it { should_not be_running }
+    end
+    describe service('httpd') do
+      it { should_not be_enabled }
+      it { should_not be_running }
+    end
+    describe service('puppetserver') do
+      it { should_not be_enabled }
+      it { should_not be_running }
+    end
+
+    it_behaves_like 'basic working puppetmaster'
+
+  end
+end