Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Full UTF-8 goodness. Hopefully those types of errors are gone for good.
  • Loading branch information
Travis Reeder committed Mar 22, 2010
1 parent eb3cade commit 698dacf
Show file tree
Hide file tree
Showing 7 changed files with 155 additions and 67 deletions.
4 changes: 2 additions & 2 deletions VERSION.yml
@@ -1,5 +1,5 @@
---
:major: 2
:minor: 2
:patch: 6
:minor: 3
:patch: 0

6 changes: 3 additions & 3 deletions aws.gemspec
Expand Up @@ -5,11 +5,11 @@

Gem::Specification.new do |s|
s.name = %q{aws}
s.version = "2.2.6"
s.version = "2.3.0"

s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Travis Reeder", "Chad Arimura", "RightScale"]
s.date = %q{2010-02-15}
s.date = %q{2010-03-22}
s.description = %q{AWS Ruby Library for interfacing with Amazon Web Services.}
s.email = %q{travis@appoxy.com}
s.extra_rdoc_files = [
Expand Down Expand Up @@ -37,7 +37,7 @@ Gem::Specification.new do |s|
s.homepage = %q{http://github.com/appoxy/aws/}
s.rdoc_options = ["--charset=UTF-8"]
s.require_paths = ["lib"]
s.rubygems_version = %q{1.3.5}
s.rubygems_version = %q{1.3.6}
s.summary = %q{AWS Ruby Library for interfacing with Amazon Web Services.}
s.test_files = [
"test/acf/test_helper.rb",
Expand Down
129 changes: 112 additions & 17 deletions lib/awsbase/right_awsbase.rb
Expand Up @@ -86,24 +86,116 @@ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, u
string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
# sign the string
signature = escape_sig(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
"#{canonical_string}&Signature=#{signature}"
ret = "#{canonical_string}&Signature=#{signature}"
# puts 'full=' + ret.inspect
ret
end

HEX = [
"%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07",
"%08", "%09", "%0A", "%0B", "%0C", "%0D", "%0E", "%0F",
"%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17",
"%18", "%19", "%1A", "%1B", "%1C", "%1D", "%1E", "%1F",
"%20", "%21", "%22", "%23", "%24", "%25", "%26", "%27",
"%28", "%29", "%2A", "%2B", "%2C", "%2D", "%2E", "%2F",
"%30", "%31", "%32", "%33", "%34", "%35", "%36", "%37",
"%38", "%39", "%3A", "%3B", "%3C", "%3D", "%3E", "%3F",
"%40", "%41", "%42", "%43", "%44", "%45", "%46", "%47",
"%48", "%49", "%4A", "%4B", "%4C", "%4D", "%4E", "%4F",
"%50", "%51", "%52", "%53", "%54", "%55", "%56", "%57",
"%58", "%59", "%5A", "%5B", "%5C", "%5D", "%5E", "%5F",
"%60", "%61", "%62", "%63", "%64", "%65", "%66", "%67",
"%68", "%69", "%6A", "%6B", "%6C", "%6D", "%6E", "%6F",
"%70", "%71", "%72", "%73", "%74", "%75", "%76", "%77",
"%78", "%79", "%7A", "%7B", "%7C", "%7D", "%7E", "%7F",
"%80", "%81", "%82", "%83", "%84", "%85", "%86", "%87",
"%88", "%89", "%8A", "%8B", "%8C", "%8D", "%8E", "%8F",
"%90", "%91", "%92", "%93", "%94", "%95", "%96", "%97",
"%98", "%99", "%9A", "%9B", "%9C", "%9D", "%9E", "%9F",
"%A0", "%A1", "%A2", "%A3", "%A4", "%A5", "%A6", "%A7",
"%A8", "%A9", "%AA", "%AB", "%AC", "%AD", "%AE", "%AF",
"%B0", "%B1", "%B2", "%B3", "%B4", "%B5", "%B6", "%B7",
"%B8", "%B9", "%BA", "%BB", "%BC", "%BD", "%BE", "%BF",
"%C0", "%C1", "%C2", "%C3", "%C4", "%C5", "%C6", "%C7",
"%C8", "%C9", "%CA", "%CB", "%CC", "%CD", "%CE", "%CF",
"%D0", "%D1", "%D2", "%D3", "%D4", "%D5", "%D6", "%D7",
"%D8", "%D9", "%DA", "%DB", "%DC", "%DD", "%DE", "%DF",
"%E0", "%E1", "%E2", "%E3", "%E4", "%E5", "%E6", "%E7",
"%E8", "%E9", "%EA", "%EB", "%EC", "%ED", "%EE", "%EF",
"%F0", "%F1", "%F2", "%F3", "%F4", "%F5", "%F6", "%F7",
"%F8", "%F9", "%FA", "%FB", "%FC", "%FD", "%FE", "%FF"
]
TO_REMEMBER = 'AZaz09 -_.!~*\'()'
ASCII = {} # {'A'=>65, 'Z'=>90, 'a'=>97, 'z'=>122, '0'=>48, '9'=>57, ' '=>32, '-'=>45, '_'=>95, '.'=>}
TO_REMEMBER.each_char do |c| #unpack("c*").each do |c|
ASCII[c] = c.unpack("c")[0]
end
# puts 'ascii=' + ASCII.inspect

# Escape a string accordingly Amazon rulles
# http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
def self.amz_escape(param)

param = param.to_s
# param = param.force_encoding("UTF-8")

e = "x" # escape2(param.to_s)
# puts 'ESCAPED=' + e.inspect


#return CGI.escape(param.to_s).gsub("%7E", "~").gsub("+", "%20") # from: http://umlaut.rubyforge.org/svn/trunk/lib/aws_product_sign.rb

#param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
# '%' + $1.unpack('H2' * $1.size).join('%').upcase
#end
e = CGI.escape(param.to_s)
e = e.gsub("%7E", "~")
e = e.gsub("+", "%20")
e = e.gsub("*", "%2A")

end
# puts 'e in=' + e.inspect
# converter = Iconv.new('ASCII', 'UTF-8')
# e = converter.iconv(e) #.unpack('U*').select{ |cp| cp < 127 }.pack('U*')
# puts 'e out=' + e.inspect

e2 = CGI.escape(param)
e2 = e2.gsub("%7E", "~")
e2 = e2.gsub("+", "%20")
e2 = e2.gsub("*", "%2A")

# puts 'E2=' + e2.inspect
# puts e == e2.to_s

e2

end

def self.escape2(s)
# home grown
ret = ""
s.unpack("U*") do |ch|
# puts 'ch=' + ch.inspect
if ASCII['A'] <= ch && ch <= ASCII['Z'] # A to Z
ret << ch
elsif ASCII['a'] <= ch && ch <= ASCII['z'] # a to z
ret << ch
elsif ASCII['0'] <= ch && ch <= ASCII['9'] # 0 to 9
ret << ch
elsif ch == ASCII[' '] # space
ret << "%20" # "+"
elsif ch == ASCII['-'] || ch == ASCII['_'] || ch == ASCII['.'] || ch == ASCII['~']
ret << ch
elsif ch <= 0x007f # other ascii
ret << HEX[ch]
elsif ch <= 0x07FF # non-ascii
ret << HEX[0xc0 | (ch >> 6)]
ret << HEX[0x80 | (ch & 0x3F)]
else
ret << HEX[0xe0 | (ch >> 12)]
ret << HEX[0x80 | ((ch >> 6) & 0x3F)]
ret << HEX[0x80 | (ch & 0x3F)]
end

end
ret

end

def self.escape_sig(raw)
e = CGI.escape(raw)
Expand Down Expand Up @@ -275,6 +367,9 @@ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, ho
def generate_request2(aws_access_key, aws_secret_key, action, api_version, lib_params, user_params={}) #:nodoc:
# remove empty params from request
user_params.delete_if {|key, value| value.nil? }
# user_params.each_pair do |k,v|
# user_params[k] = v.force_encoding("UTF-8")
# end
#params_string = params.to_a.collect{|key,val| key + "=#{CGI::escape(val.to_s)}" }.join("&")
# prepare service data
service = lib_params[:service]
Expand Down Expand Up @@ -571,10 +666,10 @@ def hash_params(prefix, list) #:nodoc:
end


# Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
# web services raise this type of error.
# Attribute inherited by RuntimeError:
# message - the text of the error, generally as returned by AWS in its XML response.
# Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
# web services raise this type of error.
# Attribute inherited by RuntimeError:
# message - the text of the error, generally as returned by AWS in its XML response.
class AwsError < RuntimeError

# either an array of errors where each item is itself an array of [code, message]),
Expand Down Expand Up @@ -647,7 +742,7 @@ def self.system_error?(e)

end

# Simplified version
# Simplified version
class AwsError2 < RuntimeError
# Request id (if exists)
attr_reader :request_id
Expand Down Expand Up @@ -851,7 +946,7 @@ def check(request) #:nodoc:
end


#-----------------------------------------------------------------
#-----------------------------------------------------------------

class RightSaxParserCallback #:nodoc:
def self.include_callback
Expand Down Expand Up @@ -1018,9 +1113,9 @@ def tagtext(text)
end
end

#-----------------------------------------------------------------
# PARSERS: Errors
#-----------------------------------------------------------------
#-----------------------------------------------------------------
# PARSERS: Errors
#-----------------------------------------------------------------

#<Error>
# <Code>TemporaryRedirect</Code>
Expand Down Expand Up @@ -1056,8 +1151,8 @@ def reset
end
end

# Dummy parser - does nothing
# Returns the original params back
# Dummy parser - does nothing
# Returns the original params back
class RightDummyParser # :nodoc:
attr_accessor :result

Expand Down
3 changes: 2 additions & 1 deletion lib/sdb/right_sdb_interface.rb
Expand Up @@ -169,7 +169,8 @@ def escape(value)
# Convert a Ruby language value to a SDB value by replacing Ruby nil with the user's chosen string representation of nil.
# Non-nil values are unaffected by this filter.
def ruby_to_sdb(value)
value.nil? ? @nil_rep : value
# puts "value #{value} is frozen? #{value.frozen?}"
value.nil? ? @nil_rep : ((value.frozen? || !value.is_a?(String)) ? value : value.force_encoding("UTF-8"))
end

# Convert a SDB value to a Ruby language value by replacing the user's chosen string representation of nil with Ruby nil.
Expand Down
8 changes: 4 additions & 4 deletions test/sdb/test_active_sdb.rb
Expand Up @@ -92,13 +92,13 @@ def test_04_find_all
ids = clients.map{|client| client.id }[0..1]
assert_equal @clients.size + 1, clients.size
# retrieve all presidents (must find: Bush, Putin, Medvedev)
assert_equal 3, Client.find(:all, :conditions => ["[?=?]",'post','president']).size
assert_equal 3, Client.find(:all, :conditions => ["post=?",'president']).size
# retrieve all russian presidents (must find: Putin, Medvedev)
assert_equal 2, Client.find(:all, :conditions => ["['post'=?] intersection ['country'=?]",'president', 'Russia']).size
assert_equal 2, Client.find(:all, :conditions => ["post=? and country=?",'president', 'Russia']).size
# retrieve all russian presidents and all women (must find: Putin, Medvedev, 2 Maries and Sandy)
assert_equal 5, Client.find(:all, :conditions => ["['post'=?] intersection ['country'=?] union ['gender'=?]",'president', 'Russia','female']).size
assert_equal 5, Client.find(:all, :conditions => ["post=? and country=? or gender=?",'president', 'Russia','female']).size
# find all rissian presidents Bushes
assert_equal 0, Client.find(:all, :conditions => ["['post'=?] intersection ['country'=?] intersection ['name'=?]",'president', 'Russia','Bush']).size
assert_equal 0, Client.find(:all, :conditions => ["post=? and country=? and name=?",'president', 'Russia','Bush']).size
# --- find by ids
# must find 1 rec (by rec id) and return it
assert_equal ids.first, Client.find(ids.first).id
Expand Down
71 changes: 31 additions & 40 deletions test/sdb/test_right_sdb.rb
Expand Up @@ -103,45 +103,16 @@ def test_07_delete_item
wait SDB_DELAY, 'after adding attributes'
# get attributes ('girls' and 'vodka' must be there)
values = @sdb.get_attributes(@domain, @item)[:attributes]['Volodya'].to_a.sort
assert_equal values, ['girls', 'vodka']
assert_equal ['girls', 'vodka'], values
# delete an item
@sdb.delete_attributes @domain, @item
sleep 1
# get attributes (values must be empty)
values = @sdb.get_attributes(@domain, @item)[:attributes]['Volodya']
assert_equal values, nil
end

def test_08_query
# not applicable anymore
end

def test_09_signature_version_0
sdb = Aws::SdbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '0')
item = 'toys'
# TODO: need to change the below test. I think Juergen's intention was to include some umlauts in the values
# put attributes
# mhhh... Not sure how to translate this: hölzchehn klötzchen grÃŒnspan buße... Lets assume this is:
attributes = { 'Jurgen' => %w{kitten puppy chickabiddy piglet} }
assert sdb.put_attributes(@domain, item, attributes)
wait SDB_DELAY, 'after putting attributes'
# get attributes
values = sdb.get_attributes(@domain, item)[:attributes]['Jurgen'].to_a.sort
# compare to original list
assert_equal values, attributes['Jurgen'].sort
# check that the request has correct signature version
assert sdb.last_request.path.include?('SignatureVersion=0')
assert_nil values
end

def test_10_signature_version_1
sdb = Aws::SdbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '1')
domains = nil
assert_nothing_thrown "Failed to use signature V1" do
domains = sdb.list_domains
end
assert domains
end

def test_11_signature_version_1
def test_11_signature_version_2
sdb = Aws::SdbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '2')
domains = nil
assert_nothing_thrown "Failed to use signature V2" do
Expand All @@ -150,26 +121,46 @@ def test_11_signature_version_1
assert domains
end

def test_12_array_of_attrs
def test_12_unicode

# This was creating a bad signature
s = ''
File.open("unicode.txt", "r") { |f|
s = f.read
}
# s = s.force_encoding("UTF-8")
puts 's=' + s.inspect
puts "encoding? " + s.encoding.name
# s = s.encode("ASCII")
# todo: I'm thinking just iterate through characters and swap out ones that aren't in ascii range.
@sdb.put_attributes @domain, @item, {"badname"=>[s]}
sleep 1
value = @sdb.get_attributes(@domain, @item)[:attributes]['badname'][0]
puts 'value=' + value.inspect
assert value == s
end

def test_15_array_of_attrs
item = 'multiples'
assert_nothing_thrown "Failed to put multiple attrs" do
@sdb.put_attributes(@domain, item, {:one=>1, :two=>2, :three=>3})
end
end

def test_13_zero_len_attrs
def test_16_zero_len_attrs
item = 'zeroes'
assert_nothing_thrown "Failed to put zero-length attributes" do
@sdb.put_attributes(@domain, item, {:one=>"", :two=>"", :three=>""})
end
end

def test_14_nil_attrs
def test_17_nil_attrs
item = 'nils'
res = nil
assert_nothing_thrown do
@sdb.put_attributes(@domain, item, {:one=>nil, :two=>nil, :three=>'chunder'})
end
sleep 1
assert_nothing_thrown do
res = @sdb.get_attributes(@domain, item)
end
Expand All @@ -178,7 +169,7 @@ def test_14_nil_attrs
assert_not_nil(res[:attributes]['three'][0])
end

def test_15_url_escape
def test_18_url_escape
item = 'urlescapes'
content = {:a=>"one & two & three",
:b=>"one ? two / three"}
Expand All @@ -189,7 +180,7 @@ def test_15_url_escape
assert_equal(content[:b], res[:attributes]['b'][0])
end

def test_16_put_attrs_by_post
def test_19_put_attrs_by_post
item = 'reqgirth'
i = 0
sa = ""
Expand All @@ -200,12 +191,12 @@ def test_16_put_attrs_by_post
@sdb.put_attributes(@domain, item, {:a => sa, :b => sa, :c => sa, :d => sa, :e => sa})
end

def test_20_query_with_atributes
def test_21_query_with_atributes
# not applicable anymore
end

# Keep this test last, because it deletes the domain...
def test_21_delete_domain
def test_40_delete_domain
assert @sdb.delete_domain(@domain), 'delete_domain fail'
wait SDB_DELAY, 'after domain deletion'
# check that domain does not exist
Expand Down
1 change: 1 addition & 0 deletions test/sdb/unicode.txt
@@ -0,0 +1 @@
Helmut Fischer GmbH Institut für Elektronik und Messtechnik

0 comments on commit 698dacf

Please sign in to comment.