From 4dc4d67bafa6a07b74a20ac4738c625e2c324f86 Mon Sep 17 00:00:00 2001 From: Marcel Molina Date: Sun, 19 Apr 2009 16:32:06 -0700 Subject: [PATCH] Full 1.9 compatibility (all tests passing against 1.9 & 1.8.6). --- CHANGELOG | 4 +-- lib/aws/s3.rb | 5 ++- lib/aws/s3/authentication.rb | 2 +- lib/aws/s3/base.rb | 22 +++++++----- lib/aws/s3/connection.rb | 2 +- lib/aws/s3/extensions.rb | 68 ++++++++++++++++++++++++++---------- lib/aws/s3/logging.rb | 15 ++++---- lib/aws/s3/object.rb | 2 +- lib/aws/s3/version.rb | 6 ++-- test/extensions_test.rb | 37 ++++++++++++-------- test/logging_test.rb | 2 +- test/remote/test_helper.rb | 5 ++- test/test_helper.rb | 5 ++- 13 files changed, 113 insertions(+), 62 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3528a8a..0b4e349 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,6 @@ head: -0.5.1: - -0.5.1: +- Full 1.9 compatibility (all tests passing against 1.9 & 1.8.6). Thanks to [David (dvdplm@gmail.com), Cyril David (cyx.ucron@gmail.com)] 0.5.1: diff --git a/lib/aws/s3.rb b/lib/aws/s3.rb index fc7d3f0..af115fb 100644 --- a/lib/aws/s3.rb +++ b/lib/aws/s3.rb @@ -1,4 +1,3 @@ -require 'base64' require 'cgi' require 'uri' require 'openssl' @@ -11,7 +10,7 @@ $:.unshift(File.dirname(__FILE__)) require 's3/extensions' require_library_or_gem 'builder' unless defined? Builder -require_library_or_gem 'mime/types' unless defined? MIME::Types +require_library_or_gem 'mime/types', 'mime-types' unless defined? MIME::Types require 's3/base' require 's3/version' @@ -43,7 +42,7 @@ include AWS::S3::BitTorrent end -require_library_or_gem 'xmlsimple' unless defined? XmlSimple +require_library_or_gem 'xmlsimple', 'xml-simple' unless defined? XmlSimple # If libxml is installed, we use the FasterXmlSimple library, that provides most of the functionality of XmlSimple # except it uses the xml/libxml library for xml parsing (rather than REXML). If libxml isn't installed, we just fall back on # XmlSimple. diff --git a/lib/aws/s3/authentication.rb b/lib/aws/s3/authentication.rb index dbb0a85..0efbc7a 100644 --- a/lib/aws/s3/authentication.rb +++ b/lib/aws/s3/authentication.rb @@ -69,7 +69,7 @@ def canonical_string def encoded_canonical digest = OpenSSL::Digest::Digest.new('sha1') - b64_hmac = Base64.encode64(OpenSSL::HMAC.digest(digest, secret_access_key, canonical_string)).strip + b64_hmac = [OpenSSL::HMAC.digest(digest, secret_access_key, canonical_string)].pack("m").strip url_encode? ? CGI.escape(b64_hmac) : b64_hmac end diff --git a/lib/aws/s3/base.rb b/lib/aws/s3/base.rb index 78a3611..63abafa 100644 --- a/lib/aws/s3/base.rb +++ b/lib/aws/s3/base.rb @@ -74,8 +74,13 @@ def request(verb, path, options = {}, body = nil, attempts = 0, &block) # Once in a while, a request to S3 returns an internal error. A glitch in the matrix I presume. Since these # errors are few and far between the request method will rescue InternalErrors the first three times they encouter them # and will retry the request again. Most of the time the second attempt will work. - rescue *retry_exceptions - attempts == 3 ? raise : (attempts += 1; retry) + rescue InternalError, RequestTimeout + if attempts == 3 + raise + else + attempts += 1 + retry + end end [:get, :post, :put, :delete, :head].each do |verb| @@ -173,10 +178,6 @@ class << self def bucket_name(name) name || current_bucket end - - def retry_exceptions - [InternalError, RequestTimeout] - end class RequestOptions < Hash #:nodoc: attr_reader :options, :verb @@ -226,9 +227,12 @@ def request(*args, &block) def method_missing(method, *args, &block) case - when attributes.has_key?(method.to_s): attributes[method.to_s] - when attributes.has_key?(method): attributes[method] - else super + when attributes.has_key?(method.to_s) + attributes[method.to_s] + when attributes.has_key?(method) + attributes[method] + else + super end end end diff --git a/lib/aws/s3/connection.rb b/lib/aws/s3/connection.rb index 1c4557b..5793dcd 100644 --- a/lib/aws/s3/connection.rb +++ b/lib/aws/s3/connection.rb @@ -7,7 +7,7 @@ def connect(options = {}) end def prepare_path(path) - path = path.remove_extended unless path.utf8? + path = path.remove_extended unless path.valid_utf8? URI.escape(path) end end diff --git a/lib/aws/s3/extensions.rb b/lib/aws/s3/extensions.rb index 1e9aba2..6a9bc75 100644 --- a/lib/aws/s3/extensions.rb +++ b/lib/aws/s3/extensions.rb @@ -26,9 +26,16 @@ def to_normalized_options! end class String - def previous! - self[-1] -= 1 - self + if RUBY_VERSION <= '1.9' + def previous! + self[-1] -= 1 + self + end + else + def previous! + self[-1] = (self[-1].ord - 1).chr + self + end end def previous @@ -48,17 +55,34 @@ def underscore tr("-", "_").downcase end unless public_method_defined? :underscore - def utf8? - scan(/[^\x00-\xa0]/u) { |s| s.unpack('U') } - true - rescue ArgumentError - false + if RUBY_VERSION >= '1.9' + def valid_utf8? + dup.force_encoding('UTF-8').valid_encoding? + end + else + def valid_utf8? + scan(Regexp.new('[^\x00-\xa0]', nil, 'u')) { |s| s.unpack('U') } + true + rescue ArgumentError + false + end end # All paths in in S3 have to be valid unicode so this takes care of - # cleaning up any strings that aren't valid utf-8 according to String#utf8? - def remove_extended! - gsub!(/[\x80-\xFF]/) { "%02X" % $&[0] } + # cleaning up any strings that aren't valid utf-8 according to String#valid_utf8? + if RUBY_VERSION >= '1.9' + def remove_extended! + sanitized_string = '' + each_byte do |byte| + character = byte.chr + sanitized_string << character if character.ascii_only? + end + sanitized_string + end + else + def remove_extended! + gsub!(/[\x80-\xFF]/) { "%02X" % $&[0] } + end end def remove_extended @@ -75,11 +99,11 @@ def coerce(string) def coerce case self - when 'true': true - when 'false': false + when 'true'; true + when 'false'; false # Don't coerce numbers that start with zero - when /^[1-9]+\d*$/: Integer(self) - when datetime_format: Time.parse(self) + when /^[1-9]+\d*$/; Integer(self) + when datetime_format; Time.parse(self) else self end @@ -103,10 +127,15 @@ def to_header module Kernel def __method__(depth = 0) caller[depth][/`([^']+)'/, 1] - end #if RUBY_VERSION < '1.8.7' + end if RUBY_VERSION < '1.8.7' + + def __called_from__ + caller[1][/`([^']+)'/, 1] + end if RUBY_VERSION > '1.8.7' def memoize(reload = false, storage = nil) - storage = "@#{storage || __method__(1)}" + current_method = RUBY_VERSION >= '1.8.7' ? __called_from__ : __method__(1) + storage = "@#{storage || current_method}" if reload instance_variable_set(storage, nil) else @@ -117,7 +146,10 @@ def memoize(reload = false, storage = nil) instance_variable_set(storage, yield) end - def require_library_or_gem(library) + def require_library_or_gem(library, gem_name = nil) + if RUBY_VERSION >= '1.9' + gem(gem_name || library, '>=0') + end require library rescue LoadError => library_not_installed begin diff --git a/lib/aws/s3/logging.rb b/lib/aws/s3/logging.rb index 2504f02..4ffc5e4 100644 --- a/lib/aws/s3/logging.rb +++ b/lib/aws/s3/logging.rb @@ -97,8 +97,14 @@ def initialize(log_object) #:nodoc: end # Returns the lines for the log. Each line is wrapped in a Log::Line. - def lines - log.value.map {|line| Line.new(line)} + if RUBY_VERSION >= '1.8.7' + def lines + log.value.lines.map {|line| Line.new(line)} + end + else + def lines + log.value.map {|line| Line.new(line)} + end end memoized :lines @@ -158,10 +164,7 @@ def #{name} # Time.parse doesn't like %d/%B/%Y:%H:%M:%S %z so we have to transform it unfortunately def typecast_time(datetime) #:nodoc: - month = datetime[/[a-z]+/i] - month_names = [nil, "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] - datetime.sub!(%r|^(\w{2})/(\w{3})|, '\2/\1') - datetime.sub!(month, month_names.index(month).to_s) + datetime.sub!(%r|^(\w{2})/(\w{3})/(\w{4})|, '\2 \1 \3') datetime.sub!(':', ' ') Time.parse(datetime) end diff --git a/lib/aws/s3/object.rb b/lib/aws/s3/object.rb index 89858f6..77a60df 100644 --- a/lib/aws/s3/object.rb +++ b/lib/aws/s3/object.rb @@ -168,7 +168,7 @@ def find(key, bucket = nil) # We need to ensure the key doesn't have extended characters but not uri escape it before doing the lookup and comparing since if the object exists, # the key on S3 will have been normalized - key = key.remove_extended unless key.utf8? + key = key.remove_extended unless key.valid_utf8? bucket = Bucket.find(bucket_name(bucket), :marker => key.previous, :max_keys => 1) # If our heuristic failed, trigger a NoSuchKey exception if (object = bucket.objects.first) && object.key == key diff --git a/lib/aws/s3/version.rb b/lib/aws/s3/version.rb index 094e9e1..e112659 100644 --- a/lib/aws/s3/version.rb +++ b/lib/aws/s3/version.rb @@ -2,9 +2,9 @@ module AWS module S3 module VERSION #:nodoc: MAJOR = '0' - MINOR = '5' - TINY = '1' - BETA = Time.now.to_i.to_s + MINOR = '6' + TINY = '0' + BETA = nil # Time.now.to_i.to_s end Version = [VERSION::MAJOR, VERSION::MINOR, VERSION::TINY, VERSION::BETA].compact * '.' diff --git a/test/extensions_test.rb b/test/extensions_test.rb index 677f7c5..846d76e 100644 --- a/test/extensions_test.rb +++ b/test/extensions_test.rb @@ -65,14 +65,14 @@ def test_to_header end end - def test_utf8? - assert !"318597/620065/GTL_75\24300_A600_A610.zip".utf8? - assert "318597/620065/GTL_75£00_A600_A610.zip".utf8? + def test_valid_utf8? + assert !"318597/620065/GTL_75\24300_A600_A610.zip".valid_utf8? + assert "318597/620065/GTL_75£00_A600_A610.zip".valid_utf8? end def test_remove_extended - assert "318597/620065/GTL_75\24300_A600_A610.zip".remove_extended.utf8? - assert "318597/620065/GTL_75£00_A600_A610.zip".remove_extended.utf8? + assert "318597/620065/GTL_75\24300_A600_A610.zip".remove_extended.valid_utf8? + assert "318597/620065/GTL_75£00_A600_A610.zip".remove_extended.valid_utf8? end end @@ -139,7 +139,7 @@ def test___method___depth assert_equal 'foo', b.foo assert_equal 'bar', b.bar end -end +end if RUBY_VERSION < '1.8.7' class ModuleExtensionsTest < Test::Unit::TestCase class Foo @@ -166,10 +166,10 @@ def setup end def test_memoize - assert !@instance.instance_variables.include?('@foo') + assert !instance_variables_of(@instance).include?('@foo') cached_result = @instance.foo assert_equal cached_result, @instance.foo - assert @instance.instance_variables.include?('@foo') + assert instance_variables_of(@instance).include?('@foo') assert_equal cached_result, @instance.send(:instance_variable_get, :@foo) assert_not_equal cached_result, new_cache = @instance.foo(:reload) assert_equal new_cache, @instance.foo @@ -177,21 +177,21 @@ def test_memoize end def test_customizing_memoize_storage - assert !@instance.instance_variables.include?('@bar') - assert !@instance.instance_variables.include?('@baz') + assert !instance_variables_of(@instance).include?('@bar') + assert !instance_variables_of(@instance).include?('@baz') cached_result = @instance.bar - assert !@instance.instance_variables.include?('@bar') - assert @instance.instance_variables.include?('@baz') + assert !instance_variables_of(@instance).include?('@bar') + assert instance_variables_of(@instance).include?('@baz') assert_equal cached_result, @instance.bar assert_equal cached_result, @instance.send(:instance_variable_get, :@baz) assert_nil @instance.send(:instance_variable_get, :@bar) end def test_memoized - assert !@instance.instance_variables.include?('@quux') + assert !instance_variables_of(@instance).include?('@quux') cached_result = @instance.quux assert_equal cached_result, @instance.quux - assert @instance.instance_variables.include?('@quux') + assert instance_variables_of(@instance).include?('@quux') assert_equal cached_result, @instance.send(:instance_variable_get, :@quux) assert_not_equal cached_result, new_cache = @instance.quux(:reload) assert_equal new_cache, @instance.quux @@ -220,6 +220,15 @@ def test_constant_setting assert_equal 'bar', some_module::FOO assert_equal 'bar', some_module.foo end + + private + # For 1.9 compatibility + def instance_variables_of(object) + object.instance_variables.map do |instance_variable| + instance_variable.to_s + end + end + end class AttributeProxyTest < Test::Unit::TestCase diff --git a/test/logging_test.rb b/test/logging_test.rb index c4c0259..ffee79b 100644 --- a/test/logging_test.rb +++ b/test/logging_test.rb @@ -58,7 +58,7 @@ def test_field_accessors expected_results = { :owner => Owner.new('id' => 'bb2041a25975c3d4ce9775fe9e93e5b77a6a9fad97dc7e00686191f3790b13f1'), :bucket => 'marcel', - :time => Time.parse('11/14/2006 06:36:48 +0000'), + :time => Time.parse('Nov 14 2006 06:36:48 +0000'), :remote_ip => '67.165.183.125', :request_id => '8B5297D428A05432', :requestor => Owner.new('id' => 'bb2041a25975c3d4ce9775fe9e93e5b77a6a9fad97dc7e00686191f3790b13f1'), diff --git a/test/remote/test_helper.rb b/test/remote/test_helper.rb index d93846b..70a6356 100644 --- a/test/remote/test_helper.rb +++ b/test/remote/test_helper.rb @@ -2,7 +2,10 @@ require 'uri' $:.unshift File.dirname(__FILE__) + '/../../lib' require 'aws/s3' -require_library_or_gem 'breakpoint' +begin + require_library_or_gem 'breakpoint' +rescue LoadError +end TEST_BUCKET = 'aws-s3-tests' TEST_FILE = File.dirname(__FILE__) + '/test_file.data' diff --git a/test/test_helper.rb b/test/test_helper.rb index aab3518..04d51d7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -3,7 +3,10 @@ require 'aws/s3' require File.dirname(__FILE__) + '/mocks/fake_response' require File.dirname(__FILE__) + '/fixtures' -require_library_or_gem 'ruby-debug' +begin + require_library_or_gem 'ruby-debug' +rescue LoadError +end require_library_or_gem 'flexmock' require_library_or_gem 'flexmock/test_unit'