(MODULES-7698) Fix OSX agent upgrades

Previously, OSX agent upgrades tried installing the new agent via. a
Puppet package resource using the pkgdmg provider. Unfortunately, this
does not work for several reasons.

* pkgdmg is unversioned. This means that whenever it installs a package,
it outputs a /var/db/.puppet_pkgdmg_installed_<package_name> file
indicating that the package is installed. For our case, we'd have a
/var/db/.puppet_pkgdmg_installed_puppet_agent file. This means that
upgrades of the form "V1 => V2 => V3" (e.g. 2018.1.3 => 2018.1.4 =>
2018.5) fail at the "V2 => V3" path because that file would still exist
from the "V1 => V2" upgrade. Thus, pkgdmg would be tricked into thinking
that the V3 agent was installed since it queries that file to see if the
package is already installed on the system.

* The installer stops the Puppet service in the middle of the
installation which interrupts the current Puppet run if that run was
triggered by the Puppet service. The installer proceeds to restart
the service in the post-installation step. This can trigger a Puppet run
while the installer's still running, which leads to weird race conditions
like an infinite loop or Puppet installing the upgraded agent several times
(typically twice) in successful cases.

This commit takes care of these issues by following the same pattern we
use for our Solaris 10 agent upgrades. Specifically, our Puppet run
performs the upgrade in the background. The background process waits for
the current Puppet run to exit before initiating the upgrade.

manifests/install.pp

@@ -55,7 +55,7 @@
     $_unzipped_package_name = regsubst($package_file_name, '\.gz$', '')
     $adminfile = '/opt/puppetlabs/packages/solaris-noask'
     $sourcefile = "/opt/puppetlabs/packages/${_unzipped_package_name}"
-    $install_script = "solaris_install.sh.erb"
+    $install_script = 'solaris_install.sh.erb'
 
     # Puppet prior to 5.0 would not use a separate process contract when forking from the Puppet
     # service. That resulted in service-initiated upgrades failing because trying to remove or
@@ -121,15 +121,24 @@
       refreshonly => true,
     }
   } elsif $::operatingsystem == 'Darwin' and $::macosx_productversion_major =~ /^10\.(9|10|11|12|13)/ {
-    contain puppet_agent::install::remove_packages
+    if $old_packages or $puppet_agent::aio_upgrade_required {
+      $install_script = 'osx_install.sh.erb'
 
-    # package provider does not provide 'versionable'
-    package { $::puppet_agent::package_name:
-      ensure          => 'present',
-      provider        => 'pkgdmg',
-      source          => "/opt/puppetlabs/packages/${package_file_name}",
-      require         => Class['puppet_agent::install::remove_packages'],
-      install_options => $install_options,
+      contain puppet_agent::install::remove_packages
+
+      $_logfile = "${::env_temp_variable}/osx_install.log"
+      notice("Puppet install log file at ${_logfile}")
+
+      $_installsh = "${::env_temp_variable}/osx_install.sh"
+      file { "${_installsh}":
+        ensure  => file,
+        mode    => '0755',
+        content => template('puppet_agent/do_install.sh.erb'),
+        require => Class['Puppet_agent::Install::Remove_packages']
+      }
+      -> exec { 'osx_install script':
+        command => "${_installsh} ${::puppet_agent_pid} 2>&1 > ${_logfile} &",
+      }
     }
   } elsif $::osfamily == 'windows' {
     # Prevent re-running the batch install

spec/classes/puppet_agent_osfamily_darwin_spec.rb

@@ -33,6 +33,8 @@
     :architecture                => 'x86_64',
     :servername                  => 'master.example.vm',
     :clientcert                  => 'foo.example.vm',
+    :env_temp_variable           => '/tmp',
+    :puppet_agent_pid            => 42
   }
 
   describe 'unsupported environment' do
@@ -57,17 +59,18 @@
           let(:facts) do
             facts.merge({
               :is_pe                       => true,
+              :aio_agent_version           => '1.0.4',
               :platform_tag                => tag,
-              :macosx_productversion_major => osmajor,
+              :macosx_productversion_major => osmajor
             })
           end
 
           it { should compile.with_all_deps }
           it { is_expected.to contain_file('/opt/puppetlabs') }
           it { is_expected.to contain_file('/opt/puppetlabs/packages') }
           it { is_expected.to contain_file("/opt/puppetlabs/packages/puppet-agent-#{package_version}-1.osx#{osmajor}.dmg") }
-          it { is_expected.to contain_package('puppet-agent').with_ensure('present') }
-          it { is_expected.to contain_package('puppet-agent').with_source("/opt/puppetlabs/packages/puppet-agent-#{package_version}-1.osx#{osmajor}.dmg") }
+          it { is_expected.to contain_file('/tmp/osx_install.sh') }
+          it { is_expected.to contain_exec('osx_install script') }
           it { is_expected.to contain_class('puppet_agent::install::remove_packages') }
           it { is_expected.to contain_class('puppet_agent::install::remove_packages_osx') }
           it { is_expected.to contain_class("puppet_agent::osfamily::darwin") }

templates/osx_install.sh.erb

@@ -0,0 +1,5 @@
+mountpoint="$(mktemp -d)"
+package_file_path="<%= scope.lookupvar('puppet_agent::params::local_packages_dir') %>/<%= @package_file_name %>"
+/usr/bin/hdiutil attach "${package_file_path}" -nobrowse -readonly -mountpoint "${mountpoint?}"
+/usr/sbin/installer -pkg ${mountpoint?}/puppet-agent-*-installer.pkg -target /
+/usr/bin/hdiutil detach "${mountpoint?}"