Permalink
Switch branches/tags
Nothing to show
Find file
Fetching contributors…
Cannot retrieve contributors at this time
executable file 178 lines (163 sloc) 6.89 KB
#!/usr/bin/env ruby
# This software code is made available "AS IS" without warranties of any
# kind. You may copy, display, modify and redistribute the software
# code either by itself or as incorporated into your code; provided that
# you do not remove any proprietary notices. Your use of this software
# code is at your own risk and you waive any claim against the author
# with respect to your use of this software code.
# (c) 2007 s3sync.net
#
module S3sync
$AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
$AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
$AWS_S3_HOST = (ENV["AWS_S3_HOST"] or "s3.amazonaws.com")
$HTTP_PROXY_HOST = ENV["HTTP_PROXY_HOST"]
$HTTP_PROXY_PORT = ENV["HTTP_PROXY_PORT"]
$HTTP_PROXY_USER = ENV["HTTP_PROXY_USER"]
$HTTP_PROXY_PASSWORD = ENV["HTTP_PROXY_PASSWORD"]
$SSL_CERT_DIR = ENV["SSL_CERT_DIR"]
$SSL_CERT_FILE = ENV["SSL_CERT_FILE"]
$S3SYNC_RETRIES = (ENV["S3SYNC_RETRIES"] or 100).to_i # number of errors to tolerate
$S3SYNC_WAITONERROR = (ENV["S3SYNC_WAITONERROR"] or 30).to_i # seconds
$S3SYNC_NATIVE_CHARSET = (ENV["S3SYNC_NATIVE_CHARSET"] or "ISO-8859-1")
$AWS_CALLING_FORMAT = (ENV["AWS_CALLING_FORMAT"] or "REGULAR")
require 'S3'
require 'HTTPStreaming'
require 'S3encoder'
CGI::exemptSlashesInEscape = true
CGI::usePercent20InEscape = true
CGI::useUTF8InEscape = true
CGI::nativeCharacterEncoding = $S3SYNC_NATIVE_CHARSET
require 'S3_s3sync_mod'
$S3syncRetriesLeft = $S3SYNC_RETRIES.to_i
def S3sync.s3trySetup
# ---------- CONNECT ---------- #
$S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
$S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
if $S3syncOptions['--ssl']
if $SSL_CERT_DIR
$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
$S3syncConnection.ca_path = $SSL_CERT_DIR
elsif $SSL_CERT_FILE
$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
$S3syncConnection.ca_file = $SSL_CERT_FILE
end
end
end
def S3sync.s3urlSetup
$S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
$S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
$S3syncGenerator.expires_in = $S3syncOptions['--expires-in']
end
def S3sync.S3tryConnect(bucket, host='')
$S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD)
end
def S3sync.S3try(command, bucket, *args)
if(not $S3syncHttp or (bucket != $S3syncLastBucket))
$stderr.puts "Creating new connection" if $S3syncOptions['--debug']
$S3syncLastBucket = bucket
while $S3syncRetriesLeft > 0 do
begin
S3sync.S3tryConnect(bucket)
break
rescue Errno::ECONNRESET => e
$stderr.puts "Connection reset: #{e}"
rescue Errno::ECONNABORTED => e
$stderr.puts "Connection aborted: #{e}"
rescue Errno::ETIMEDOUT => e
$stderr.puts "Connection timed out: #{e}"
rescue Timeout::Error => e
$stderr.puts "Connection timed out: #{e}"
end
$S3syncRetriesLeft -= 1
$stderr.puts "#{$S3syncRetriesLeft} retries left, sleeping for #{$S3SYNC_WAITONERROR} seconds"
Kernel.sleep $S3SYNC_WAITONERROR.to_i
end
end
result = nil
delim = $,
$,=' '
while $S3syncRetriesLeft > 0 do
$stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug']
forceRetry = false
now = false
hush = false
begin
result = $S3syncConnection.send(command, bucket, *args)
rescue Errno::EPIPE => e
forceRetry = true
$stderr.puts "Broken pipe: #{e}"
rescue Errno::ECONNRESET => e
forceRetry = true
$stderr.puts "Connection reset: #{e}"
rescue Errno::ECONNABORTED => e
forceRetry = true
$stderr.puts "Connection aborted: #{e}"
rescue Errno::ETIMEDOUT => e
forceRetry = true
$stderr.puts "Connection timed out: #{e}"
rescue Timeout::Error => e
forceRetry = true
$stderr.puts "Connection timed out: #{e}"
rescue EOFError => e
# i THINK this is happening like a connection reset
forceRetry = true
$stderr.puts "EOF error: #{e}"
rescue OpenSSL::SSL::SSLError => e
forceRetry = true
$stderr.puts "SSL Error: #{e}"
rescue NoMethodError => e
# we get this when using --progress, and the local item is something unreadable
$stderr.puts "Null stream error: #{e}"
break
end
if forceRetry
# kill and reset connection when we receive a non-50x yet retry-able error
S3sync.S3tryConnect(bucket)
end
begin
debug("Response code: #{result.http_response.code}")
break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry)
if result.http_response.code.to_i == 301
$stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN"
elsif result.http_response.code.to_i == 307
# move to the other host
host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1]
$stderr.puts("Temporary Redirect to: " + host)
debug("Host: " + host)
S3sync.S3tryConnect(bucket, host)
$S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below
forceRetry = true
now = true
hush = true
else
$stderr.puts "S3 command failed:\n#{command} #{args}"
$stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n"
debug(result.http_response.body)
end
# only retry 500's, per amazon
break unless ((500...600).include? result.http_response.code.to_i) or forceRetry
rescue NoMethodError
debug("No result available")
end
$S3syncRetriesLeft -= 1
$stderr.puts "#{$S3syncRetriesLeft} retries left, sleeping for #{$S3SYNC_WAITONERROR} seconds" unless hush
Kernel.sleep $S3SYNC_WAITONERROR.to_i unless now
end
if $S3syncRetriesLeft <= 0
$stderr.puts "Ran out of retries; operations did not complete!"
end
$, = delim
result
end
def S3sync.S3url(command, bucket, *args)
S3sync.s3urlSetup() unless $S3syncGenerator
result = nil
delim = $,
$,=' '
$stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug']
result = $S3syncGenerator.send(command, bucket, *args)
$, = delim
result
end
end #module