Merge pull request #203 from puppetlabs/RE-14819

(RE-14819) Retry failed curls before trying to gunzip them

lib/beaker-pe/install/pe_utils.rb

@@ -378,8 +378,8 @@ def fetch_pe_on_unix(host, opts)
           local = File.directory?(path)
           filename = "#{host['dist']}"
           if local
-            extension = File.exists?("#{path}/#{filename}.tar.gz") ? ".tar.gz" : ".tar"
-            if not File.exists?("#{path}/#{filename}#{extension}")
+            extension = File.exist?("#{path}/#{filename}.tar.gz") ? ".tar.gz" : ".tar"
+            unless File.exist?("#{path}/#{filename}#{extension}")
               raise "attempting installation on #{host}, #{path}/#{filename}#{extension} does not exist"
             end
             scp_to host, "#{path}/#{filename}#{extension}", "#{host['working_dir']}/#{filename}#{extension}"
@@ -404,16 +404,23 @@ def fetch_pe_on_unix(host, opts)
             if host['platform'] =~ /eos/
               host.get_remote_file("#{path}/#{filename}#{extension}")
             else
-              unpack = 'tar -xvf -'
-              unpack = extension =~ /gz/ ? 'gunzip | ' + unpack  : unpack
               if opts[:fetch_local_then_push_to_host]
                 fetch_and_push_pe(host, path, filename, extension)
                 command_file_push = 'cat '
               else
                 curlopts = opts[:use_proxy] ? "--proxy #{opts[:proxy_hostname]}:3128 " : ""
-                command_file_push = "curl -L #{curlopts}#{path}/"
+                command_file_push = "curl --fail --location --output #{filename}#{extension} #{curlopts}#{path}/"
               end
-              on host, "cd #{host['working_dir']}; #{command_file_push}#{filename}#{extension} | #{unpack}"
+
+              retry_requirements = {
+                desired_exit_codes: [0],
+                max_retries: 3,
+                verbose: 'true'
+              }
+              fetch_tarball_command = "cd #{host['working_dir']}; #{command_file_push}#{filename}#{extension}"
+              retry_on(host, fetch_tarball_command, retry_requirements)
+              on host, "cd #{host['working_dir']}; tar -xvf #{filename}#{extension}"
+
               gpg_key_overwrite(host, 'tarball')
             end
           end

spec/beaker-pe/install/pe_utils_spec.rb

@@ -1063,13 +1063,14 @@ def slice_installer_options(host)
   describe 'fetch_pe' do
 
     it 'can push a local PE .tar.gz to a host and unpack it' do
-      allow( File ).to receive( :directory? ).and_return( true ) #is local
-      allow( File ).to receive( :exists? ).and_return( true ) #is a .tar.gz
+      allow( File ).to receive( :directory? ).and_return( true )
+      allow( File ).to receive( :exists? ).and_return( true )
+      allow( File ).to receive( :exist? ).and_return( true )
       unixhost['pe_dir'] = '/local/file/path'
       allow( subject ).to receive( :scp_to ).and_return( true )
 
       path = unixhost['pe_dir']
-      filename = "#{ unixhost['dist'] }"
+      filename = unixhost['dist']
       extension = '.tar.gz'
       expect( subject ).to receive( :scp_to ).with( unixhost, "#{ path }/#{ filename }#{ extension }", "#{ unixhost['working_dir'] }/#{ filename }#{ extension }" ).once
       expect( subject ).to receive( :on ).with( unixhost, /gunzip/ ).once
@@ -1079,61 +1080,69 @@ def slice_installer_options(host)
 
     it 'can download a PE .tar from a URL to a host and unpack it' do
       allow( File ).to receive( :directory? ).and_return( false ) #is not local
-      allow( subject ).to receive( :link_exists? ) do |arg|
-        if arg =~ /.tar.gz/ #there is no .tar.gz link, only a .tar
-          false
-        else
-          true
-        end
-      end
-      allow( subject ).to receive( :on ).and_return( true )
+      allow( subject ).to receive( :link_exists? ) { |arg| arg !~ /.tar.gz/ }
+      zero_exit_code_mock = Object.new
+      allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
+      allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )
 
+      unixhost['pe_dir'] = '/local/file/path'
       path = unixhost['pe_dir']
-      filename = "#{ unixhost['dist'] }"
+      filename = unixhost['dist']
       extension = '.tar'
-      expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; curl -L #{ path }/#{ filename }#{ extension } | tar -xvf -" ).once
+      expect( subject )
+        .to receive( :on )
+              .with( unixhost, "cd #{ unixhost['working_dir'] }; curl --fail --location --output #{ filename }#{ extension } #{ path }/#{ filename }#{ extension }", anything ).once
       subject.fetch_pe( [unixhost], {} )
     end
 
     it 'can download a PE .tar from a URL to #fetch_and_push_pe' do
-      allow( File ).to receive( :directory? ).and_return( false ) #is not local
-      allow( subject ).to receive( :link_exists? ) do |arg|
-        if arg =~ /.tar.gz/ #there is no .tar.gz link, only a .tar
-          false
-        else
-          true
-        end
-      end
-      allow( subject ).to receive( :on ).and_return( true )
+      allow( File ).to receive( :directory? ).and_return( false )
+      allow( subject ).to receive( :link_exists? ) { |arg| arg !~ /.tar.gz/ }
 
-      filename = "#{ unixhost['dist'] }"
+      zero_exit_code_mock = Object.new
+      allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
+      allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )
+
+      filename = unixhost['dist']
       extension = '.tar'
-      expect( subject ).to receive( :fetch_and_push_pe ).with( unixhost, anything, filename, extension ).once
-      expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension } | tar -xvf -" ).once
+      expect( subject )
+        .to receive( :fetch_and_push_pe )
+              .with( unixhost, anything, filename, extension ).once
+      expect( subject )
+        .to receive( :on )
+              .with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension }", anything )
+              .once
       subject.fetch_pe( [unixhost], {:fetch_local_then_push_to_host => true} )
     end
 
     it 'can download a PE .tar.gz from a URL to a host and unpack it' do
       allow( File ).to receive( :directory? ).and_return( false ) #is not local
       allow( subject ).to receive( :link_exists? ).and_return( true ) #is a tar.gz
-      allow( subject ).to receive( :on ).and_return( true )
+
+      zero_exit_code_mock = Object.new
+      allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
+      allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )
 
       path = unixhost['pe_dir']
-      filename = "#{ unixhost['dist'] }"
+      filename = unixhost['dist']
       extension = '.tar.gz'
-      expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; curl -L #{ path }/#{ filename }#{ extension } | gunzip | tar -xvf -" ).once
+      expect( subject )
+        .to receive( :on )
+              .with( unixhost, "cd #{ unixhost['working_dir'] }; curl --fail --location --output #{ filename }#{ extension } #{ path }/#{ filename }#{ extension }", anything ).once
       subject.fetch_pe( [unixhost], {} )
     end
 
     it 'can download a PE .tar.gz from a URL to #fetch_and_push_pe' do
-      allow( File ).to receive( :directory? ).and_return( false ) #is not local
-      allow( subject ).to receive( :link_exists? ).and_return( true ) #is a tar.gz
-      allow( subject ).to receive( :on ).and_return( true )
+      allow( File ).to receive( :directory? ).and_return( false )
+      allow( subject ).to receive( :link_exists? ).and_return( true )
+      zero_exit_code_mock = Object.new
+      allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
+      allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )
 
-      filename = "#{ unixhost['dist'] }"
+      filename = unixhost['dist']
       extension = '.tar.gz'
       expect( subject ).to receive( :fetch_and_push_pe ).with( unixhost, anything, filename, extension ).once
-      expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension } | gunzip | tar -xvf -" ).once
+      expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension }", anything ).once
       subject.fetch_pe( [unixhost], {:fetch_local_then_push_to_host => true} )
     end
 
@@ -1367,7 +1376,7 @@ def slice_installer_options(host)
 
       allow(subject).to receive(:stop_agent_on).and_return(true)
       expect(subject).to receive(:stop_agent_on).with([mono_master, pe_postgres], :run_in_parallel => true).once
-      
+
       allow(subject).to receive(:on).with(mono_master, "puppet agent -t", :acceptable_exit_codes=>[0, 2]).exactly(3).times
       allow(subject).to receive(:on).with(pe_postgres, "puppet agent -t", :acceptable_exit_codes=> [0, 2]).once