diff --git a/CHANGELOG.md b/CHANGELOG.md index 553ebeb..b5bc7ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## Change Log +### v0.3.4 + +- Fix handling gzip/deflate response +- Change the default accept-encoding to 'identity' +- Allow setting custom HTTP headers in get_object + ### v0.3.3 - Fix object key problem in batch_delete diff --git a/lib/aliyun/oss/bucket.rb b/lib/aliyun/oss/bucket.rb index a881950..d53f279 100644 --- a/lib/aliyun/oss/bucket.rb +++ b/lib/aliyun/oss/bucket.rb @@ -236,6 +236,8 @@ def put_object(key, opts = {}, &block) # * :if_unmodified_since (Time) 指定如果object从这个时间后再无修改,则下载 # * :if_match_etag (String) 指定如果object的etag等于这个值,则下载 # * :if_unmatch_etag (String) 指定如果object的etag不等于这个值,则下载 + # @option opts [Hash] :headers 指定请求的HTTP Header,不区分大小 + # 写。这里指定的值会覆盖通过`:range`和`:condition`设置的值。 # @option opts [Hash] :rewrite 指定下载object时Server端返回的响应头部字段的值 # * :content_type (String) 指定返回的响应中Content-Type的值 # * :content_language (String) 指定返回的响应中Content-Language的值 @@ -472,10 +474,7 @@ def resumable_upload(key, file, opts = {}, &block) args[:content_type] ||= get_content_type(file) args[:content_type] ||= get_content_type(key) - - unless cpt_file = args[:cpt_file] - cpt_file = get_cpt_file(file) - end + cpt_file = args[:cpt_file] || get_cpt_file(file) Multipart::Upload.new( @protocol, options: args, @@ -507,6 +506,8 @@ def resumable_upload(key, file, opts = {}, &block) # 则:cpt_file会被忽略。 # @option opts [Hash] :condition 指定下载object需要满足的条件, # 同 {#get_object} + # @option opts [Hash] :headers 指定请求的HTTP Header,不区分大小 + # 写。这里指定的值会覆盖通过`:condition`设置的值。 # @option opts [Hash] :rewrite 指定下载object时Server端返回的响 # 应头部字段的值,同 {#get_object} # @yield [Float] 如果调用的时候传递了block,则会将下载进度交由 @@ -529,10 +530,7 @@ def resumable_download(key, file, opts = {}, &block) args[:content_type] ||= get_content_type(file) args[:content_type] ||= get_content_type(key) - - unless cpt_file = args[:cpt_file] - cpt_file = get_cpt_file(file) - end + cpt_file = args[:cpt_file] || get_cpt_file(file) Multipart::Download.new( @protocol, options: args, diff --git a/lib/aliyun/oss/download.rb b/lib/aliyun/oss/download.rb index cc00d69..b1188bf 100644 --- a/lib/aliyun/oss/download.rb +++ b/lib/aliyun/oss/download.rb @@ -179,7 +179,8 @@ def download_part(p) part_file = get_part_file(p) File.open(part_file, 'w') do |w| @protocol.get_object( - bucket, object, :range => p[:range]) { |chunk| w.write(chunk) } + bucket, object, + @options.merge(range: p[:range])) { |chunk| w.write(chunk) } end sync_update_part(p.merge(done: true, md5: get_file_md5(part_file))) diff --git a/lib/aliyun/oss/http.rb b/lib/aliyun/oss/http.rb index c4645c6..4fe2496 100644 --- a/lib/aliyun/oss/http.rb +++ b/lib/aliyun/oss/http.rb @@ -32,6 +32,7 @@ module OSS class HTTP DEFAULT_CONTENT_TYPE = 'application/octet-stream' + DEFAULT_ACCEPT_ENCODING = 'identity' STS_HEADER = 'x-oss-security-token' OPEN_TIMEOUT = 10 READ_TIMEOUT = 120 @@ -156,8 +157,25 @@ def handle_response(r, &block) r.read_body else # streaming read body on success - r.read_body do |chunk| - yield RestClient::Request.decode(r['content-encoding'], chunk) + encoding = r['content-encoding'] + if encoding == 'gzip' + stream = StreamWriter.new { |s| r.read_body { |chunk| s << chunk } } + reader = Zlib::GzipReader.new(stream) + yield reader.read(16 * 1024) until reader.eof? + elsif encoding == 'deflate' + begin + stream = Zlib::Inflate.new + r.read_body { |chunk| stream << chunk } + stream.finish { |chunk| yield chunk } + rescue Zlib::DataError + # No luck with Zlib decompression. Let's try with raw deflate, + # like some broken web servers do. + stream = Zlib::Inflate.new(-Zlib::MAX_WBITS) + r.read_body { |chunk| stream << chunk } + stream.finish { |chunk| yield chunk } + end + else + r.read_body { |chunk| yield chunk } end end end @@ -210,6 +228,7 @@ def do_request(verb, resources = {}, http_options = {}, &block) headers['user-agent'] = get_user_agent headers['date'] = Time.now.httpdate headers['content-type'] ||= DEFAULT_CONTENT_TYPE + headers['accept-encoding'] ||= DEFAULT_ACCEPT_ENCODING headers[STS_HEADER] = @config.sts_token if @config.sts_token if body = http_options[:body] diff --git a/lib/aliyun/oss/protocol.rb b/lib/aliyun/oss/protocol.rb index 8c28663..7fe4284 100644 --- a/lib/aliyun/oss/protocol.rb +++ b/lib/aliyun/oss/protocol.rb @@ -718,6 +718,9 @@ def list_objects(bucket_name, opts = {}) # specified # * :if_unmatch_etag (String) get the object if its etag # doesn't match specified + # @option opts [Hash] :headers custom HTTP headers, case + # insensitive. Headers specified here will overwrite `:condition` + # and `:range` # @option opts [Hash] :rewrite response headers to rewrite # * :content_type (String) the Content-Type header # * :content_language (String) the Content-Language header @@ -739,6 +742,7 @@ def get_object(bucket_name, object_name, opts = {}, &block) headers = {} headers['range'] = get_bytes_range(range) if range headers.merge!(get_conditions(conditions)) if conditions + headers.merge!(to_lower_case(opts[:headers])) if opts.key?(:headers) sub_res = {} if rewrites diff --git a/lib/aliyun/version.rb b/lib/aliyun/version.rb index 5637e9d..046e387 100644 --- a/lib/aliyun/version.rb +++ b/lib/aliyun/version.rb @@ -2,6 +2,6 @@ module Aliyun - VERSION = "0.3.3" + VERSION = "0.3.4" end # Aliyun diff --git a/tests/test_content_encoding.rb b/tests/test_content_encoding.rb new file mode 100644 index 0000000..8c2d09f --- /dev/null +++ b/tests/test_content_encoding.rb @@ -0,0 +1,59 @@ +require 'minitest/autorun' +require 'yaml' +$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__)) +require 'aliyun/oss' +require 'zlib' + +class TestContentEncoding < Minitest::Test + def setup + Aliyun::Common::Logging.set_log_level(Logger::DEBUG) + conf_file = '~/.oss.yml' + conf = YAML.load(File.read(File.expand_path(conf_file))) + client = Aliyun::OSS::Client.new( + :endpoint => conf['endpoint'], + :cname => conf['cname'], + :access_key_id => conf['access_key_id'], + :access_key_secret => conf['access_key_secret']) + @bucket = client.get_bucket(conf['bucket']) + + @prefix = "tests/content_encoding/" + end + + def get_key(k) + "#{@prefix}#{k}" + end + + def test_gzip_encoding + key = get_key('gzip') + File.open('/tmp/x', 'w') do |f| + 1000.times { f.write 'hello world' * 1024 } + end + + @bucket.put_object( + key, file: '/tmp/x', content_type: 'text/plain') + + @bucket.get_object( + key, file: '/tmp/y', headers: {'accept-encoding': 'gzip'}) + + assert File.exist?('/tmp/y') + diff = `diff /tmp/x /tmp/y` + assert diff.empty? + end + + def test_deflate_encoding + key = get_key('deflate') + File.open('/tmp/x', 'w') do |f| + 1000.times { f.write 'hello world' * 1024 } + end + + @bucket.put_object( + key, file: '/tmp/x', content_type: 'text/plain') + + @bucket.get_object( + key, file: '/tmp/y', headers: {'accept-encoding': 'deflate'}) + + assert File.exist?('/tmp/y') + diff = `diff /tmp/x /tmp/y` + assert diff.empty? + end +end