From da3426afde82c314149179f19a80ad26ba1b1dae Mon Sep 17 00:00:00 2001 From: Daniel Dreier Date: Sat, 15 Nov 2014 14:36:16 -0800 Subject: [PATCH 1/3] Switch default beaker nodeset to Debian Ubuntu is still available but the default is Debian 7 --- spec/acceptance/nodesets/default.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/acceptance/nodesets/default.yml b/spec/acceptance/nodesets/default.yml index f7df2cc..f9cf0c9 100644 --- a/spec/acceptance/nodesets/default.yml +++ b/spec/acceptance/nodesets/default.yml @@ -1,10 +1,10 @@ HOSTS: - ubuntu-server-12042-x64: + debian-73-x64: roles: - master - platform: ubuntu-12.04-amd64 - box : ubuntu-server-12042-x64-vbox4210-nocm - box_url : http://puppet-vagrant-boxes.puppetlabs.com/ubuntu-server-12042-x64-vbox4210-nocm.box + platform: debian-7-amd64 + box : debian-73-x64-virtualbox-nocm + box_url : http://puppet-vagrant-boxes.puppetlabs.com/debian-73-x64-virtualbox-nocm.box hypervisor : vagrant CONFIG: log_level: debug From 1c8e133fa2fe4584b684074a62c5cb41cb37a893 Mon Sep 17 00:00:00 2001 From: Daniel Dreier Date: Sat, 15 Nov 2014 14:37:36 -0800 Subject: [PATCH 2/3] Separate out server acceptance tests passenger, thin, unicorn, and webrick can be tested independently This implies differences to how tests are run, so documentation was updated The advantage here is no chance of bleed-over between test cases, whereas the previous way required imperfect automated cleanup between test cases. This also permits testing one thing at a time to save a little bit of time. Unfortunately, it also means that testing all cases requires considerably longer because a new VM must be provisioned for each case. --- CONTRIBUTING.md | 38 ++++-- spec/acceptance/agent_spec.rb | 2 +- spec/acceptance/passenger_server_spec.rb | 25 ++++ spec/acceptance/server_spec.rb | 164 ----------------------- spec/acceptance/thin_server_spec.rb | 40 ++++++ spec/acceptance/unicorn_server_spec.rb | 45 +++++++ spec/acceptance/webrick_server_spec.rb | 34 +++++ spec/spec_helper_acceptance.rb | 3 + 8 files changed, 178 insertions(+), 173 deletions(-) create mode 100644 spec/acceptance/passenger_server_spec.rb delete mode 100644 spec/acceptance/server_spec.rb create mode 100644 spec/acceptance/thin_server_spec.rb create mode 100644 spec/acceptance/unicorn_server_spec.rb create mode 100644 spec/acceptance/webrick_server_spec.rb diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 345f048..c71085a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,25 +1,47 @@ Testing ======= -Tests on the puppet-puppet module can be run via `rake`. +Puppet-puppet is tested via beaker and rspec-puppet. Rspec-puppet tests are run +automatically for pull requests via travis-ci, or can be triggered manually +by running `rake spec` after doing a `bundle install`. -#### Beaker acceptance tests -``` -BEAKER_destroy=no BEAKER_provision=onpass rake acceptance +Beaker tests are run slightly differently for puppet-puppet than for other +beaker-tested projects because the test suite should be run independently for +each test case to prevent contamination between environments. For example, +running the apache-passenger based puppet master test case will cause obvious +conflicts if the nginx-unicorn puppet master is subsequently built on the same +virtual machine. Rspec tags are used to accomplish this in the test cases. + +Running beaker tests for unicorn puppetmaster on the default nodeset (debian 7): +```bash +rspec . --tag servertype:unicorn --pattern "spec/acceptance/**/*_spec.rb" ``` -Run beaker acceptance tests on debian 7: +Same as above, but only only destroying the VM if the build is successful, to +help troubleshooting: ``` -BEAKER_set=debian-73-x64 BEAKER_destroy=onpass BEAKER_provision=no rake acceptance +BEAKER_destroy=onpass BEAKER_provision=yes rspec . --tag servertype:unicorn --pattern "spec/acceptance/**/*_spec.rb" ``` -See spec/acceptance/nodesets for a list of possible node names; use the filename without .yml. + +Same as above, but on a different nodeset (see `spec/acceptance/nodesets` for +options): +```bash +BEAKER_set=sles-11sp1-x64 BEAKER_destroy=onpass BEAKER_provision=yes rspec . --tag servertype:unicorn --pattern "spec/acceptance/**/*_spec.rb" +``` + +The presence of an OS/distro in the nodeset list does not imply support. The +SLES example above is expected to fail most test cases but is included to lower +the bar for future contributors who want to add support for additional distros. + +The rake command includes acceptance testing tasks, but these should not be +used because they will run all of the acceptance tests on the same VM, which +is expected to fail. If a beaker test fails, you can SSH into the environment if you use BEAKER_PROVISION=onpass. The path of the vagrantfile will be `.vagrant/beaker_vagrant_files/debian-73-x64.yml` if you followed the above instructions, and slightly different if you used a different nodeset. `cd` to that directory and `vagrant ssh` to access the VM. The tests that ran are in /tmp with randomly generated filenames. -``` #### rspec-puppet tests (note that these are run automatically by travis CI on pull requests) diff --git a/spec/acceptance/agent_spec.rb b/spec/acceptance/agent_spec.rb index bbbc8b0..109bc0f 100644 --- a/spec/acceptance/agent_spec.rb +++ b/spec/acceptance/agent_spec.rb @@ -2,7 +2,7 @@ describe 'agent', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do context 'with default parameters' do - it 'should run with no errors' do + it 'should run with no errors', :agent do pp = <<-EOS class { 'puppet::agent': } EOS diff --git a/spec/acceptance/passenger_server_spec.rb b/spec/acceptance/passenger_server_spec.rb new file mode 100644 index 0000000..5954fe7 --- /dev/null +++ b/spec/acceptance/passenger_server_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper_acceptance' + +describe 'passenger server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do + context 'running on passenger' do + it 'should run with no errors', :servertype => 'passenger', :webserver => 'apache' do + pp = <<-EOS + class { "puppet::server": + servertype => 'passenger', + ca => true, + servername => $::hostname, + } + EOS + + # Run it twice and test for idempotency + apply_manifest(pp, :catch_failures => true) + expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero + end + + describe port(8140) do + it { + should be_listening + } + end + end +end diff --git a/spec/acceptance/server_spec.rb b/spec/acceptance/server_spec.rb deleted file mode 100644 index edbd078..0000000 --- a/spec/acceptance/server_spec.rb +++ /dev/null @@ -1,164 +0,0 @@ -require 'spec_helper_acceptance' -def cleanup_server() - cleanup_pp = <<-EOS - $services = [ 'apache2', 'httpd', 'puppetmaster', 'unicorn_puppetmaster' ] - service { $services: ensure => 'stopped' } - $packages = [ 'nginx', 'apache2', 'httpd', 'puppetmaster', 'puppetmaster-common', 'ruby-passenger'] - package { $packages: ensure => 'absent' } - EOS - - apply_manifest(cleanup_pp, :catch_failures => false) - apply_manifest(cleanup_pp, :catch_failures => false) - shell("killall apache2 httpd nginx puppetmaster unicorn", :acceptable_exit_codes => [0,1] ) -end - -describe 'server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do - before(:all) do - cleanup_server() - end - - context 'running on webrick/standalone' do - after(:all) do - cleanup_server() - end - it 'should run with no errors' do - pp = <<-EOS - class { "puppet::server": - servertype => 'standalone', - ca => true, - } - EOS - - # Run it twice and test for idempotency - apply_manifest(pp, :catch_failures => true) - expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero - end - - describe service('puppetmaster') do - it { - should be_enabled - } - it { - should be_running - } - end - - describe port(8140) do - it { - should be_listening - } - end - end - - context 'running on passenger' do - after(:all) do - cleanup_server() - end - - it 'should run with no errors' do - pp = <<-EOS - class { "puppet::server": - servertype => 'passenger', - ca => true, - servername => $::hostname, - } - EOS - - # Run it twice and test for idempotency - apply_manifest(pp, :catch_failures => true) - expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero - end - - describe port(8140) do - it { - should be_listening - } - end - - end - - context 'running on unicorn' do - after(:all) do - cleanup_server() - end - - it 'should run with no errors' do - pp = <<-EOS - class { "puppet::server": - servertype => 'unicorn', - ca => true, - } - EOS - - # Run it twice and test for idempotency - apply_manifest(pp, :catch_failures => true) - expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero - end - describe command('puppet agent --test --server puppet') do - its(:exit_status) { should eq 0 } - its(:stderr) { should_not match /Forbidden request:/ } - its(:stderr) { should_not match /Error:/ } - end - - describe package('nginx') do - it { - should be_installed - } - end - - describe service('nginx') do - it { - should be_enabled - } - it { - should be_running - } - end - - describe port(8140) do - it { - should be_listening - } - end - end - - context 'running on thin' do - after(:all) do - cleanup_server() - end - - it 'should run with no errors' do - pp = <<-EOS - class { "puppet::server": - servertype => 'thin', - ca => true, - } - EOS - - # Run it twice and test for idempotency - apply_manifest(pp, :catch_failures => true) - expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero - end - - describe package('thin') do - it { - should be_installed.by('gem') - } - end - - describe service('thin-puppetmaster') do - it { - should be_enabled - } - it { - should be_running - } - end - - describe port(8140) do - it { - should be_listening - } - end - end -end diff --git a/spec/acceptance/thin_server_spec.rb b/spec/acceptance/thin_server_spec.rb new file mode 100644 index 0000000..d248d13 --- /dev/null +++ b/spec/acceptance/thin_server_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper_acceptance' + +describe 'thin server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do + + context 'running on thin' do + it 'should run with no errors', :servertype => 'thin', :webserver => 'nginx' do + pp = <<-EOS + class { "puppet::server": + servertype => 'thin', + ca => true, + } + EOS + + # Run it twice and test for idempotency + apply_manifest(pp, :catch_failures => true) + expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero + end + + describe package('thin') do + it { + should be_installed.by('gem') + } + end + + describe service('thin-puppetmaster') do + it { + should be_enabled + } + it { + should be_running + } + end + + describe port(8140) do + it { + should be_listening + } + end + end +end diff --git a/spec/acceptance/unicorn_server_spec.rb b/spec/acceptance/unicorn_server_spec.rb new file mode 100644 index 0000000..6d2b5ed --- /dev/null +++ b/spec/acceptance/unicorn_server_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper_acceptance' + +describe 'server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do + context 'running on unicorn' do + it 'should run with no errors', :servertype => 'unicorn', :webserver => 'nginx' do + pp = <<-EOS + class { "puppet::server": + servertype => 'unicorn', + ca => true, + } + EOS + + # Run it twice and test for idempotency + apply_manifest(pp, :catch_failures => true) + expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero + end + describe command('puppet agent --test --server puppet') do + its(:exit_status) { should eq 0 } + its(:stderr) { should_not match /Forbidden request:/ } + its(:stderr) { should_not match /Error:/ } + end + + describe package('nginx') do + it { + should be_installed + } + end + + describe service('nginx') do + it { + should be_enabled + } + it { + should be_running + } + end + + describe port(8140) do + it { + should be_listening + } + end + end + +end diff --git a/spec/acceptance/webrick_server_spec.rb b/spec/acceptance/webrick_server_spec.rb new file mode 100644 index 0000000..05d1c96 --- /dev/null +++ b/spec/acceptance/webrick_server_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper_acceptance' + +describe 'server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do + context 'running on webrick/standalone' do + it 'should run with no errors', :server => 'webrick', :webserver => 'builtin' do + pp = <<-EOS + class { "puppet::server": + servertype => 'standalone', + ca => true, + } + EOS + + # Run it twice and test for idempotency + apply_manifest(pp, :catch_failures => true) + expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero + end + + describe service('puppetmaster') do + it { + should be_enabled + } + it { + should be_running + } + end + + describe port(8140) do + it { + should be_listening + } + end + end + +end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 36f58d3..2aca372 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -58,6 +58,9 @@ EOS on host, "echo \"#{puppetfile}\" > /etc/puppet/Puppetfile" on host, "cd /etc/puppet; r10k puppetfile install" + on host, "mkdir -p /etc/puppet/environments/production/modules" + on host, "puppet config set --section master environmentpath '$confdir/environments'" + on host, "puppet config set --section master basemodulepath '$confdir/modules'" proj_root = File.expand_path(File.join(File.dirname(__FILE__), '..')) puppet_module_install(:source => proj_root, :module_name => 'puppet') From 2cec963c826bc86d95c72b0f594cd4c64fd26ad3 Mon Sep 17 00:00:00 2001 From: Daniel Dreier Date: Sat, 15 Nov 2014 23:45:45 -0800 Subject: [PATCH 3/3] Improve acceptance tests Add an actual agent run to each test, to confirm that the master is able to serve requests in some basic capacity. Would have caught the "unicorn / nginx support broken" issue where ssl headers weren't passed right. Also moves some of the tests into common blocks to reduce duplication. --- spec/acceptance/passenger_server_spec.rb | 30 +++++++++++------ spec/acceptance/thin_server_spec.rb | 24 +++++--------- spec/acceptance/unicorn_server_spec.rb | 37 ++++----------------- spec/acceptance/webrick_server_spec.rb | 42 +++++++++++++++--------- spec/spec_helper_acceptance.rb | 29 ++++++++++++++++ 5 files changed, 90 insertions(+), 72 deletions(-) diff --git a/spec/acceptance/passenger_server_spec.rb b/spec/acceptance/passenger_server_spec.rb index 5954fe7..f4002ec 100644 --- a/spec/acceptance/passenger_server_spec.rb +++ b/spec/acceptance/passenger_server_spec.rb @@ -1,13 +1,13 @@ require 'spec_helper_acceptance' describe 'passenger server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do - context 'running on passenger' do - it 'should run with no errors', :servertype => 'passenger', :webserver => 'apache' do + context 'running on passenger', :servertype => 'passenger', :webserver => 'apache' do + it 'should run with no errors' do pp = <<-EOS - class { "puppet::server": - servertype => 'passenger', - ca => true, - servername => $::hostname, + class { 'puppet::server': + servertype => 'passenger', + ca => true, + servername => $::hostname, } EOS @@ -15,11 +15,21 @@ class { "puppet::server": apply_manifest(pp, :catch_failures => true) expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero end + it_behaves_like 'basic working puppetmaster' - describe port(8140) do - it { - should be_listening - } + # sanity checks to ensure the passenger setup doesn't bring in other services + describe service('nginx') do + it { should_not be_enabled } + it { should_not be_running } end + describe service('puppetmaster') do + it { should_not be_enabled } + it { should_not be_running } + end + describe service('puppetserver') do + it { should_not be_enabled } + it { should_not be_running } + end + end end diff --git a/spec/acceptance/thin_server_spec.rb b/spec/acceptance/thin_server_spec.rb index d248d13..30b18b7 100644 --- a/spec/acceptance/thin_server_spec.rb +++ b/spec/acceptance/thin_server_spec.rb @@ -2,8 +2,8 @@ describe 'thin server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do - context 'running on thin' do - it 'should run with no errors', :servertype => 'thin', :webserver => 'nginx' do + context 'running on thin', :servertype => 'thin', :webserver => 'nginx' do + it 'should run with no errors' do pp = <<-EOS class { "puppet::server": servertype => 'thin', @@ -17,24 +17,16 @@ class { "puppet::server": end describe package('thin') do - it { - should be_installed.by('gem') - } + it { should be_installed.by('gem') } end describe service('thin-puppetmaster') do - it { - should be_enabled - } - it { - should be_running - } + it { should be_enabled } + it { should be_running } end - describe port(8140) do - it { - should be_listening - } - end + it_behaves_like "basic working puppetmaster" + it_behaves_like "nginx-based webserver" + end end diff --git a/spec/acceptance/unicorn_server_spec.rb b/spec/acceptance/unicorn_server_spec.rb index 6d2b5ed..594be88 100644 --- a/spec/acceptance/unicorn_server_spec.rb +++ b/spec/acceptance/unicorn_server_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper_acceptance' describe 'server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do - context 'running on unicorn' do - it 'should run with no errors', :servertype => 'unicorn', :webserver => 'nginx' do + context 'running on unicorn', :servertype => 'unicorn', :webserver => 'nginx' do + it 'should run with no errors' do pp = <<-EOS - class { "puppet::server": - servertype => 'unicorn', - ca => true, + class { 'puppet::server': + servertype => 'unicorn', + ca => true, } EOS @@ -14,32 +14,9 @@ class { "puppet::server": apply_manifest(pp, :catch_failures => true) expect(apply_manifest(pp, :catch_failures => true).exit_code).to be_zero end - describe command('puppet agent --test --server puppet') do - its(:exit_status) { should eq 0 } - its(:stderr) { should_not match /Forbidden request:/ } - its(:stderr) { should_not match /Error:/ } - end - - describe package('nginx') do - it { - should be_installed - } - end - describe service('nginx') do - it { - should be_enabled - } - it { - should be_running - } - end + it_behaves_like "basic working puppetmaster" + it_behaves_like "nginx-based webserver" - describe port(8140) do - it { - should be_listening - } - end end - end diff --git a/spec/acceptance/webrick_server_spec.rb b/spec/acceptance/webrick_server_spec.rb index 05d1c96..5c6eca7 100644 --- a/spec/acceptance/webrick_server_spec.rb +++ b/spec/acceptance/webrick_server_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper_acceptance' describe 'server', :unless => UNSUPPORTED_PLATFORMS.include?(fact('osfamily')) do - context 'running on webrick/standalone' do - it 'should run with no errors', :server => 'webrick', :webserver => 'builtin' do + context 'running on webrick/standalone', :server => 'webrick', :webserver => 'builtin' do + it 'should run with no errors' do pp = <<-EOS - class { "puppet::server": - servertype => 'standalone', - ca => true, + class { 'puppet::server': + servertype => 'standalone', + ca => true, } EOS @@ -16,19 +16,29 @@ class { "puppet::server": end describe service('puppetmaster') do - it { - should be_enabled - } - it { - should be_running - } + it { should be_enabled } + it { should be_running } end - describe port(8140) do - it { - should be_listening - } + # sanity checks to ensure the webrick setup doesn't bring in other services + describe service('nginx') do + it { should_not be_enabled } + it { should_not be_running } end - end + describe service('apache2') do + it { should_not be_enabled } + it { should_not be_running } + end + describe service('httpd') do + it { should_not be_enabled } + it { should_not be_running } + end + describe service('puppetserver') do + it { should_not be_enabled } + it { should_not be_running } + end + + it_behaves_like 'basic working puppetmaster' + end end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 2aca372..2eeeb1a 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -70,3 +70,32 @@ on host, "rm -rf /var/lib/puppet/ssl; puppet cert --generate $HOSTNAME" end + +shared_examples_for "basic working puppetmaster" do + describe command('puppet agent --test --server puppet') do + its(:exit_status) { should eq 0 } + its(:stderr) { should_not match /Forbidden request:/ } + its(:stderr) { should_not match /Error:/ } + end + describe port(8140) do + it { + should be_listening + } + end +end + +shared_examples_for "nginx-based webserver" do + describe package('nginx') do + it { should be_installed } + end + + describe service('nginx') do + it { should be_enabled } + it { should be_running } + end + + describe service('puppetmaster') do + it { should_not be_enabled } + it { should_not be_running } + end +end