(maint) refactor acceptance rakefile and options files

This change contains many changes to the acceptance Rakefile to put most
of the options back in the options files and use new beaker features for
creating preserved_hosts files.
It removes some of the functionality in parsing log files, but this was
all broken by beaker changes anyway.

[skip ci]

acceptance/Rakefile

@@ -1,30 +1,19 @@
 require 'rake/clean'
 require 'pp'
 require 'yaml'
-$LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
-require 'puppet/acceptance/git_utils'
-extend Puppet::Acceptance::GitUtils
 
 ONE_DAY_IN_SECS = 24 * 60 * 60
-REPO_CONFIGS_DIR = "repo-configs"
-CLEAN.include('*.tar', REPO_CONFIGS_DIR, 'merged_options.rb')
+PRESERVED_HOSTS_FILENAME = 'hosts_preserved.yml'
+SUTS_FILENAME = 'sut.log'
+CLEAN.include('*.tar', 'merged_options.rb')
 
 module HarnessOptions
 
   DEFAULTS = {
-    :type => 'git',
-    :helper => ['lib/helper.rb'],
     :tests  => ['tests'],
-    :log_level => 'debug',
-    :color => false,
-    :root_keys => true,
     :ssh => {
       :keys => ["~/.ssh/id_rsa-acceptance"],
     },
-    :xml => true,
-    :timesync => false,
-    :repo_proxy => true,
-    :add_el_extras => true,
     :preserve_hosts => 'onfail',
   }
 
@@ -36,7 +25,7 @@ module HarnessOptions
     end
 
     def get_options(file_path)
-      puts file_path
+      puts "Attempting to merge config file: #{file_path}"
       if File.exists? file_path
         options = eval(File.read(file_path), binding)
       else
@@ -69,13 +58,34 @@ module HarnessOptions
   end
 end
 
-def beaker_test(mode = :packages, options = {})
-  final_options = HarnessOptions.options(mode, options)
-
-  if mode == :git
-    final_options[:install] ||= []
-    final_options[:install] << "#{build_giturl('facter')}##{sha}"
+def hosts
+  if ENV['HOSTS']
+    ENV['HOSTS']
+  else
+    puts 'Warning: environment variable CONFIG deprecated. Please use HOSTS to match beaker options.'
+    ENV['CONFIG']
   end
+end
+
+def tests
+  ENV['TESTS'] || ENV['TEST']
+end
+
+def generate_beaker_cli_flags(options_file)
+  # the options file (including the default options might also have tests
+  #   they'll get merged with the below by beaker
+  tests_opt = "--tests=#{tests}" if tests
+
+  hosts_opt = "--hosts=#{hosts}" if hosts
+
+  overriding_options = ENV['OPTIONS']
+
+  # compact to remove the nil elements
+  ["--options-file", options_file, hosts_opt, tests_opt, overriding_options].compact
+end
+
+def write_merged_options_file(mode,options)
+  final_options = HarnessOptions.options(mode, options)
 
   options_file = 'merged_options.rb'
   File.open(options_file, 'w') do |merged|
@@ -85,88 +95,40 @@ def beaker_test(mode = :packages, options = {})
 EOS
     merged.puts(final_options.pretty_inspect)
   end
+  return options_file
+end
 
-  tests = ENV['TESTS'] || ENV['TEST']
-  tests_opt = "--tests=#{tests}" if tests
-
-  config_opt = "--hosts=#{config}" if config
-
-  overriding_options = ENV['OPTIONS']
-
-  args = ["--options-file", options_file, config_opt, tests_opt, overriding_options].compact
+def beaker_test(mode = :aio, options = {})
+  options_file = write_merged_options_file(mode,options)
+  args = generate_beaker_cli_flags(options_file)
 
   begin
     sh("beaker", *args)
   ensure
-    if (hosts_file = config || final_options[:hosts_file]) && hosts_file !~ /preserved_config/
-      cp(hosts_file, "log/latest/config.yml")
-      generate_config_for_latest_hosts if final_options[:preserve_hosts] || overriding_options =~ /--preserve-hosts/
-    end
     mv(options_file, "log/latest")
     puts "\n\n"
   end
 end
 
-def generate_config_for_latest_hosts
-  preserved_config_hash = { 'HOSTS' => {} }
-
-  config_hash = YAML.load_file('log/latest/config.yml').to_hash
-  nodes = config_hash['HOSTS'].map do |node_label,hash|
-    { :node_label => node_label, :platform => hash['platform'] }
-  end
-
-  pre_suite_log = File.read('log/latest/pre_suite-run.log')
-  nodes.each do |node_info|
-    possible_hostname_match = /^(\w+) \(#{node_info[:node_label]}\)/.match(pre_suite_log)
-    hostname = (possible_hostname_match || Array.new)[1]
-    fqdn = hostname ? "#{hostname}.delivery.puppetlabs.net" : "unknown"
-    preserved_config_hash['HOSTS'][fqdn] = {
-      'roles' => [ 'agent'],
-      'platform' => node_info[:platform],
-    }
-    preserved_config_hash['HOSTS'][fqdn]['roles'].unshift('master') if node_info[:node_label] =~ /master/
-  end
-
-  puts "\n\n"
-  pp preserved_config_hash
-  puts "\n\n"
-
-  File.open('log/latest/preserved_config.yaml', 'w') do |config_file|
-    YAML.dump(preserved_config_hash, config_file)
-  end
-rescue Errno::ENOENT => e
-  puts "Couldn't generate log #{e}"
-end
-
-def list_preserved_configurations(secs_ago = ONE_DAY_IN_SECS)
-  preserved = {}
-  Dir.glob('log/*_*').each do |dir|
-    preserved_config_path = "#{dir}/preserved_config.yaml"
-    yesterday = Time.now - secs_ago.to_i
-    if preserved_config = File.exists?(preserved_config_path)
-      directory = File.new(dir)
-      if directory.ctime > yesterday
-        hosts = []
-        preserved_config = YAML.load_file(preserved_config_path).to_hash
-        preserved_config['HOSTS'].each do |hostname,values|
-          hosts << "#{hostname}: #{values['platform']}, #{values['roles']}"
-        end
-        preserved[hosts] = directory.to_path
-      end
-    end
-  end
-  preserved.map { |k,v| [v,k] }.sort { |a,b| a[0] <=> b[0] }.reverse
-end
-
 def list_preserved_hosts(secs_ago = ONE_DAY_IN_SECS)
   hosts = Set.new
-  Dir.glob('log/**/pre*suite*run.log').each do |log|
-    yesterday = Time.now - secs_ago.to_i
-    File.open(log, 'r') do |file|
-      if file.ctime > yesterday
-        file.each_line do |line|
-          matchdata = /^(\w+) \(.*?\) \$/.match(line.encode!('UTF-8', 'UTF-8', :invalid => :replace))
-          hosts.add(matchdata[1]) if matchdata
+  Dir.glob('log/*').each do |dir|
+    unless dir =~ /^log\/latest$/
+      Dir.glob("#{dir}/*").each do |date_dir|
+        preserved_config_path = "#{date_dir}/#{SUTS_FILENAME}"
+        if preserved_config = File.exists?(preserved_config_path)
+          directory = File.new(date_dir)
+          yesterday = Time.now - secs_ago.to_i
+          if directory.ctime > yesterday
+            File.open(preserved_config_path, 'r') do |file|
+              if file.ctime > yesterday
+                file.each_line do |line|
+                  matchdata = /(\w+\.delivery\.puppetlabs\.net) \(.*?\)$/.match(line)
+                  hosts.add(matchdata[1]) if matchdata
+                end
+              end
+            end
+          end
         end
       end
     end
@@ -212,36 +174,27 @@ def destroy_preserved_hosts(hosts = nil, secs_ago = ONE_DAY_IN_SECS)
 end
 
 def print_preserved(preserved)
-  preserved.each_with_index do |entry,i|
-    puts "##{i}: #{entry[0]}"
-    entry[1].each { |h| puts "  #{h}" }
+  preserved.each do |entry|
+    puts entry
   end
 end
 
 def beaker_run_type
-  type = ENV['TYPE'] || :packages
+  type = ENV['TYPE'] || :aio
   type = type.to_sym
 end
 
 def sha
-  ENV['SHA']
-end
-
-def config
-  ENV['CONFIG']
+  ENV['SHA'] || ENV['GIT_SHA']
 end
 
 namespace :ci do
 
-  task :check_env do
-    raise(USAGE) unless sha
-  end
-
   namespace :test do
 
     USAGE = <<-EOS
-Requires commit SHA to be put under test as environment variable: SHA='<sha>'.
-Also must set CONFIG=config/nodes/foo.yaml or include it in an options.rb for Beaker.
+May specify project SHA environment variable, which can be a branch, release version number, or commit SHA.
+Also must set HOSTS=config/nodes/foo.yaml or include it in an options.rb for Beaker.
 You may set TESTS=path/to/test,and/more/tests.
 You may set additional Beaker OPTIONS='--more --options'
 If testing from git checkouts, you may optionally set the github fork to checkout from using FORK='other-puppet-fork'.
@@ -250,47 +203,45 @@ If there is a Beaker options hash in a ./local_options.rb, it will be included.
 EOS
 
     desc <<-EOS
-Run the acceptance tests through Beaker and install packages on the configuration targets.
+Run the acceptance tests through Beaker and install packages on the configuration targets (only facter versions <3.0)
 #{USAGE}
 EOS
-    task :packages => 'ci:check_env' do
-      beaker_test
+    task :packages do
+      beaker_test(:packages)
     end
 
     desc <<-EOS
 Run the acceptance tests through Beaker and install packages as part of the AIO puppet-agent installation.
 #{USAGE}
 EOS
-    task :aio => 'ci:check_env' do
+    task :aio do
       beaker_test(:aio)
     end
 
     desc <<-EOS
-Run the acceptance tests through Beaker and install from git on the configuration targets.
+DEPRECATED: use the aio method for now. Run the acceptance tests through Beaker and install from git on the configuration targets.
 #{USAGE}
 EOS
-    task :git => 'ci:check_env' do
+    task :git do
       beaker_test(:git)
     end
   end
 
-  desc "Capture the master and agent hostname from the latest log and construct a preserved_config.yaml for re-running against preserved hosts without provisioning."
-  task :extract_preserved_config do
-    generate_config_for_latest_hosts
-  end
+  desc "DEPRECATED: Beaker now does this automatically."
+  task :extract_preserved_config
 
   desc <<-EOS
 Run an acceptance test for a given node configuration and preserve the hosts.
 Defaults to a packages run, but you can set it to 'git' with TYPE='git'.
 #{USAGE}
   EOS
-  task :test_and_preserve_hosts => 'ci:check_env'  do
-    beaker_test(beaker_run_type, :preserve_hosts => 'always', :__preserve_config__ => true)
+  task :test_and_preserve_hosts do
+    beaker_test(beaker_run_type, :preserve_hosts => 'always')
   end
 
   desc "List acceptance runs from the past day which had hosts preserved."
   task :list_preserved do
-    preserved = list_preserved_configurations
+    preserved = list_preserved_hosts
     print_preserved(preserved)
   end
 
@@ -306,25 +257,21 @@ You can go back through the last SECS_AGO logs.  Default is one day ago in secs.
   end
 
   desc <<-EOS
-Rerun an acceptance test using the last captured preserved_config.yaml to skip provisioning.
+Rerun an acceptance test using hosts_preserved.yaml to skip provisioning.
 Or specify a CONFIG_NUMBER from `rake ci:list_preserved`.
 Uses the setup/rsync/pre-suite to rsync the local puppet source onto master and agent.
 You may specify an RSYNC_FILTER_FILE as well.
 You may skip purgeing and reinstalling puppet packages by including SKIP_PACKAGE_REINSTALL.
-You may skip rsyncing local puppet files over to the tests hosts by including SKIP_RSYNC.
 Defaults to a packages run, but you can set it to 'git' with TYPE='git'.
   EOS
   task :test_against_preserved_hosts do
-    config_number = (ENV['CONFIG_NUMBER'] || 0).to_i
     preserved = list_preserved_configurations
     print_preserved(preserved)
+    config_number = (ENV['CONFIG_NUMBER'] || 0).to_i
     config_path = preserved[config_number][0]
     puts "Using ##{config_number}: #{config_path}"
     beaker_test(beaker_run_type,
-      :hosts_file => "#{config_path}/preserved_config.yaml",
-      :no_provision => true,
-      :preserve_hosts => 'always',
-      :pre_suite => ['setup/rsync/pre-suite']
+      :hosts_file => "log/latest/#{PRESERVED_HOSTS_FILENAME}"
     )
   end
 end

acceptance/config/aio/options.rb

@@ -1,6 +1,10 @@
 {
-  :type => 'aio',
-  :pre_suite => [
+  :type          => 'aio',
+  :forge_host    => 'forge-aio01-petest.puppetlabs.com',
+  :load_path     => './lib/',
+  :repo_proxy    => true,
+  :add_el_extras => true,
+  :pre_suite     => [
     'setup/common/pre-suite/000-delete-puppet-when-none.rb',
     'setup/aio/pre-suite/010_Install.rb',
   ],

acceptance/config/git/options.rb

@@ -1,5 +1,10 @@
 {
-  :pre_suite => [
+  :type          => 'git',
+  :forge_host    => 'forge-aio01-petest.puppetlabs.com',
+  :load_path     => './lib/',
+  :repo_proxy    => true,
+  :add_el_extras => true,
+  :pre_suite     => [
     'setup/common/pre-suite/000-delete-puppet-when-none.rb',
     'setup/common/00_EnvSetup.rb',
     'setup/git/pre-suite/01_TestSetup.rb',

acceptance/config/packages/options.rb

@@ -1,4 +1,9 @@
 {
+  :type          => 'foss',
+  :forge_host    => 'forge-aio01-petest.puppetlabs.com',
+  :load_path     => './lib/',
+  :repo_proxy    => true,
+  :add_el_extras => true,
   :pre_suite => [
     'setup/common/pre-suite/000-delete-puppet-when-none.rb',
     'setup/common/00_EnvSetup.rb',

acceptance/setup/aio/pre-suite/010_Install.rb

@@ -7,7 +7,7 @@
 test_name "Install Packages"
 
 step "Install repositories on target machines..." do
-  sha = ENV['SHA']
+  sha = ENV['SHA'] || ENV['GIT_SHA']
   hosts.each do |host|
     install_repos_on(host, 'puppet-agent', sha, 'repo-configs')
   end