Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

(RE-14819) Retry failed curls before trying to gunzip them #203

Merged
merged 2 commits into from
Nov 2, 2022
Merged

(RE-14819) Retry failed curls before trying to gunzip them #203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
27a58e5
(RE-14819) Retry failed curls before trying to gunzip them
e-gris Sep 21, 2022
03dffc0
(RE-14819) Add back gunzip
e-gris Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions lib/beaker-pe/install/pe_utils.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,8 +378,8 @@ def fetch_pe_on_unix(host, opts)
local = File.directory?(path)
filename = "#{host['dist']}"
if local
extension = File.exists?("#{path}/#{filename}.tar.gz") ? ".tar.gz" : ".tar"
if not File.exists?("#{path}/#{filename}#{extension}")
extension = File.exist?("#{path}/#{filename}.tar.gz") ? ".tar.gz" : ".tar"
unless File.exist?("#{path}/#{filename}#{extension}")
raise "attempting installation on #{host}, #{path}/#{filename}#{extension} does not exist"
end
scp_to host, "#{path}/#{filename}#{extension}", "#{host['working_dir']}/#{filename}#{extension}"
Expand All @@ -404,16 +404,23 @@ def fetch_pe_on_unix(host, opts)
if host['platform'] =~ /eos/
host.get_remote_file("#{path}/#{filename}#{extension}")
else
unpack = 'tar -xvf -'
unpack = extension =~ /gz/ ? 'gunzip | ' + unpack : unpack
if opts[:fetch_local_then_push_to_host]
fetch_and_push_pe(host, path, filename, extension)
command_file_push = 'cat '
else
curlopts = opts[:use_proxy] ? "--proxy #{opts[:proxy_hostname]}:3128 " : ""
command_file_push = "curl -L #{curlopts}#{path}/"
command_file_push = "curl --fail --location --output #{filename}#{extension} #{curlopts}#{path}/"
end
on host, "cd #{host['working_dir']}; #{command_file_push}#{filename}#{extension} | #{unpack}"

retry_requirements = {
desired_exit_codes: [0],
max_retries: 3,
verbose: 'true'
}
fetch_tarball_command = "cd #{host['working_dir']}; #{command_file_push}#{filename}#{extension}"
retry_on(host, fetch_tarball_command, retry_requirements)
on host, "cd #{host['working_dir']}; tar -xvf #{filename}#{extension}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only thing that comes to mind is the question of whether or not we can rely on tar being able to gunzip automagically on all platforms this method would execute on. But since I think this would only be used on pe master platforms, I don't think that's an issue.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I went through exactly that same series of steps. I'm willing to fix it if I got it wrong.


gpg_key_overwrite(host, 'tarball')
end
end
Expand Down
77 changes: 43 additions & 34 deletions spec/beaker-pe/install/pe_utils_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1063,13 +1063,14 @@ def slice_installer_options(host)
describe 'fetch_pe' do

it 'can push a local PE .tar.gz to a host and unpack it' do
allow( File ).to receive( :directory? ).and_return( true ) #is local
allow( File ).to receive( :exists? ).and_return( true ) #is a .tar.gz
allow( File ).to receive( :directory? ).and_return( true )
allow( File ).to receive( :exists? ).and_return( true )
allow( File ).to receive( :exist? ).and_return( true )
unixhost['pe_dir'] = '/local/file/path'
allow( subject ).to receive( :scp_to ).and_return( true )

path = unixhost['pe_dir']
filename = "#{ unixhost['dist'] }"
filename = unixhost['dist']
extension = '.tar.gz'
expect( subject ).to receive( :scp_to ).with( unixhost, "#{ path }/#{ filename }#{ extension }", "#{ unixhost['working_dir'] }/#{ filename }#{ extension }" ).once
expect( subject ).to receive( :on ).with( unixhost, /gunzip/ ).once
Expand All @@ -1079,61 +1080,69 @@ def slice_installer_options(host)

it 'can download a PE .tar from a URL to a host and unpack it' do
allow( File ).to receive( :directory? ).and_return( false ) #is not local
allow( subject ).to receive( :link_exists? ) do |arg|
if arg =~ /.tar.gz/ #there is no .tar.gz link, only a .tar
false
else
true
end
end
allow( subject ).to receive( :on ).and_return( true )
allow( subject ).to receive( :link_exists? ) { |arg| arg !~ /.tar.gz/ }
zero_exit_code_mock = Object.new
allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )

unixhost['pe_dir'] = '/local/file/path'
path = unixhost['pe_dir']
filename = "#{ unixhost['dist'] }"
filename = unixhost['dist']
extension = '.tar'
expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; curl -L #{ path }/#{ filename }#{ extension } | tar -xvf -" ).once
expect( subject )
.to receive( :on )
.with( unixhost, "cd #{ unixhost['working_dir'] }; curl --fail --location --output #{ filename }#{ extension } #{ path }/#{ filename }#{ extension }", anything ).once
subject.fetch_pe( [unixhost], {} )
end

it 'can download a PE .tar from a URL to #fetch_and_push_pe' do
allow( File ).to receive( :directory? ).and_return( false ) #is not local
allow( subject ).to receive( :link_exists? ) do |arg|
if arg =~ /.tar.gz/ #there is no .tar.gz link, only a .tar
false
else
true
end
end
allow( subject ).to receive( :on ).and_return( true )
allow( File ).to receive( :directory? ).and_return( false )
allow( subject ).to receive( :link_exists? ) { |arg| arg !~ /.tar.gz/ }

filename = "#{ unixhost['dist'] }"
zero_exit_code_mock = Object.new
allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )

filename = unixhost['dist']
extension = '.tar'
expect( subject ).to receive( :fetch_and_push_pe ).with( unixhost, anything, filename, extension ).once
expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension } | tar -xvf -" ).once
expect( subject )
.to receive( :fetch_and_push_pe )
.with( unixhost, anything, filename, extension ).once
expect( subject )
.to receive( :on )
.with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension }", anything )
.once
subject.fetch_pe( [unixhost], {:fetch_local_then_push_to_host => true} )
end

it 'can download a PE .tar.gz from a URL to a host and unpack it' do
allow( File ).to receive( :directory? ).and_return( false ) #is not local
allow( subject ).to receive( :link_exists? ).and_return( true ) #is a tar.gz
allow( subject ).to receive( :on ).and_return( true )

zero_exit_code_mock = Object.new
allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )

path = unixhost['pe_dir']
filename = "#{ unixhost['dist'] }"
filename = unixhost['dist']
extension = '.tar.gz'
expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; curl -L #{ path }/#{ filename }#{ extension } | gunzip | tar -xvf -" ).once
expect( subject )
.to receive( :on )
.with( unixhost, "cd #{ unixhost['working_dir'] }; curl --fail --location --output #{ filename }#{ extension } #{ path }/#{ filename }#{ extension }", anything ).once
subject.fetch_pe( [unixhost], {} )
end

it 'can download a PE .tar.gz from a URL to #fetch_and_push_pe' do
allow( File ).to receive( :directory? ).and_return( false ) #is not local
allow( subject ).to receive( :link_exists? ).and_return( true ) #is a tar.gz
allow( subject ).to receive( :on ).and_return( true )
allow( File ).to receive( :directory? ).and_return( false )
allow( subject ).to receive( :link_exists? ).and_return( true )
zero_exit_code_mock = Object.new
allow(zero_exit_code_mock).to receive(:exit_code).and_return(0)
allow( subject ).to receive( :on ).and_return( zero_exit_code_mock )

filename = "#{ unixhost['dist'] }"
filename = unixhost['dist']
extension = '.tar.gz'
expect( subject ).to receive( :fetch_and_push_pe ).with( unixhost, anything, filename, extension ).once
expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension } | gunzip | tar -xvf -" ).once
expect( subject ).to receive( :on ).with( unixhost, "cd #{ unixhost['working_dir'] }; cat #{ filename }#{ extension }", anything ).once
subject.fetch_pe( [unixhost], {:fetch_local_then_push_to_host => true} )
end

Expand Down Expand Up @@ -1367,7 +1376,7 @@ def slice_installer_options(host)

allow(subject).to receive(:stop_agent_on).and_return(true)
expect(subject).to receive(:stop_agent_on).with([mono_master, pe_postgres], :run_in_parallel => true).once

allow(subject).to receive(:on).with(mono_master, "puppet agent -t", :acceptable_exit_codes=>[0, 2]).exactly(3).times
allow(subject).to receive(:on).with(pe_postgres, "puppet agent -t", :acceptable_exit_codes=> [0, 2]).once

Expand Down