Fix gzip/deflate encoding problem

CHANGELOG.md

@@ -1,5 +1,11 @@
 ## Change Log
 
+### v0.3.4
+
+- Fix handling gzip/deflate response
+- Change the default accept-encoding to 'identity'
+- Allow setting custom HTTP headers in get_object
+
 ### v0.3.3
 
 - Fix object key problem in batch_delete

lib/aliyun/oss/bucket.rb

@@ -236,6 +236,8 @@ def put_object(key, opts = {}, &block)
       #   * :if_unmodified_since (Time) 指定如果object从这个时间后再无修改，则下载
       #   * :if_match_etag (String) 指定如果object的etag等于这个值，则下载
       #   * :if_unmatch_etag (String) 指定如果object的etag不等于这个值，则下载
+      # @option opts [Hash] :headers 指定请求的HTTP Header，不区分大小
+      #  写。这里指定的值会覆盖通过`:range`和`:condition`设置的值。
       # @option opts [Hash] :rewrite 指定下载object时Server端返回的响应头部字段的值
       #   * :content_type (String) 指定返回的响应中Content-Type的值
       #   * :content_language (String) 指定返回的响应中Content-Language的值
@@ -472,10 +474,7 @@ def resumable_upload(key, file, opts = {}, &block)
 
         args[:content_type] ||= get_content_type(file)
         args[:content_type] ||= get_content_type(key)
-
-        unless cpt_file = args[:cpt_file]
-          cpt_file = get_cpt_file(file)
-        end
+        cpt_file = args[:cpt_file] || get_cpt_file(file)
 
         Multipart::Upload.new(
           @protocol, options: args,
@@ -507,6 +506,8 @@ def resumable_upload(key, file, opts = {}, &block)
       #  则:cpt_file会被忽略。
       # @option opts [Hash] :condition 指定下载object需要满足的条件，
       #  同 {#get_object}
+      # @option opts [Hash] :headers 指定请求的HTTP Header，不区分大小
+      #  写。这里指定的值会覆盖通过`:condition`设置的值。
       # @option opts [Hash] :rewrite 指定下载object时Server端返回的响
       #  应头部字段的值，同 {#get_object}
       # @yield [Float] 如果调用的时候传递了block，则会将下载进度交由
@@ -529,10 +530,7 @@ def resumable_download(key, file, opts = {}, &block)
 
         args[:content_type] ||= get_content_type(file)
         args[:content_type] ||= get_content_type(key)
-
-        unless cpt_file = args[:cpt_file]
-          cpt_file = get_cpt_file(file)
-        end
+        cpt_file = args[:cpt_file] || get_cpt_file(file)
 
         Multipart::Download.new(
           @protocol, options: args,

lib/aliyun/oss/download.rb

@@ -179,7 +179,8 @@ def download_part(p)
           part_file = get_part_file(p)
           File.open(part_file, 'w') do |w|
             @protocol.get_object(
-              bucket, object, :range => p[:range]) { |chunk| w.write(chunk) }
+              bucket, object,
+              @options.merge(range: p[:range])) { |chunk| w.write(chunk) }
           end
 
           sync_update_part(p.merge(done: true, md5: get_file_md5(part_file)))

lib/aliyun/oss/http.rb

@@ -32,6 +32,7 @@ module OSS
     class HTTP
 
       DEFAULT_CONTENT_TYPE = 'application/octet-stream'
+      DEFAULT_ACCEPT_ENCODING = 'identity'
       STS_HEADER = 'x-oss-security-token'
       OPEN_TIMEOUT = 10
       READ_TIMEOUT = 120
@@ -156,8 +157,25 @@ def handle_response(r, &block)
           r.read_body
         else
         # streaming read body on success
-          r.read_body do |chunk|
-            yield RestClient::Request.decode(r['content-encoding'], chunk)
+          encoding = r['content-encoding']
+          if encoding == 'gzip'
+            stream = StreamWriter.new { |s| r.read_body { |chunk| s << chunk } }
+            reader = Zlib::GzipReader.new(stream)
+            yield reader.read(16 * 1024) until reader.eof?
+          elsif encoding == 'deflate'
+            begin
+              stream = Zlib::Inflate.new
+              r.read_body { |chunk| stream << chunk }
+              stream.finish { |chunk| yield chunk }
+            rescue Zlib::DataError
+              # No luck with Zlib decompression. Let's try with raw deflate,
+              # like some broken web servers do.
+              stream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+              r.read_body { |chunk| stream << chunk }
+              stream.finish { |chunk| yield chunk }
+            end
+          else
+            r.read_body { |chunk| yield chunk }
           end
         end
       end
@@ -210,6 +228,7 @@ def do_request(verb, resources = {}, http_options = {}, &block)
         headers['user-agent'] = get_user_agent
         headers['date'] = Time.now.httpdate
         headers['content-type'] ||= DEFAULT_CONTENT_TYPE
+        headers['accept-encoding'] ||= DEFAULT_ACCEPT_ENCODING
         headers[STS_HEADER] = @config.sts_token if @config.sts_token
 
         if body = http_options[:body]

lib/aliyun/oss/protocol.rb

@@ -718,6 +718,9 @@ def list_objects(bucket_name, opts = {})
       #     specified
       #   * :if_unmatch_etag (String) get the object if its etag
       #     doesn't match specified
+      # @option opts [Hash] :headers custom HTTP headers, case
+      #  insensitive. Headers specified here will overwrite `:condition`
+      #  and `:range`
       # @option opts [Hash] :rewrite response headers to rewrite
       #   * :content_type (String) the Content-Type header
       #   * :content_language (String) the Content-Language header
@@ -739,6 +742,7 @@ def get_object(bucket_name, object_name, opts = {}, &block)
         headers = {}
         headers['range'] = get_bytes_range(range) if range
         headers.merge!(get_conditions(conditions)) if conditions
+        headers.merge!(to_lower_case(opts[:headers])) if opts.key?(:headers)
 
         sub_res = {}
         if rewrites

lib/aliyun/version.rb

@@ -2,6 +2,6 @@
 
 module Aliyun
 
-    VERSION = "0.3.3"
+    VERSION = "0.3.4"
 
 end # Aliyun

tests/test_content_encoding.rb

@@ -0,0 +1,59 @@
+require 'minitest/autorun'
+require 'yaml'
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+require 'aliyun/oss'
+require 'zlib'
+
+class TestContentEncoding < Minitest::Test
+  def setup
+    Aliyun::Common::Logging.set_log_level(Logger::DEBUG)
+    conf_file = '~/.oss.yml'
+    conf = YAML.load(File.read(File.expand_path(conf_file)))
+    client = Aliyun::OSS::Client.new(
+      :endpoint => conf['endpoint'],
+      :cname => conf['cname'],
+      :access_key_id => conf['access_key_id'],
+      :access_key_secret => conf['access_key_secret'])
+    @bucket = client.get_bucket(conf['bucket'])
+
+    @prefix = "tests/content_encoding/"
+  end
+
+  def get_key(k)
+    "#{@prefix}#{k}"
+  end
+
+  def test_gzip_encoding
+    key = get_key('gzip')
+    File.open('/tmp/x', 'w') do |f|
+      1000.times { f.write 'hello world' * 1024 }
+    end
+
+    @bucket.put_object(
+      key, file: '/tmp/x', content_type: 'text/plain')
+
+    @bucket.get_object(
+      key, file: '/tmp/y', headers: {'accept-encoding': 'gzip'})
+
+    assert File.exist?('/tmp/y')
+    diff = `diff /tmp/x /tmp/y`
+    assert diff.empty?
+  end
+
+  def test_deflate_encoding
+    key = get_key('deflate')
+    File.open('/tmp/x', 'w') do |f|
+      1000.times { f.write 'hello world' * 1024 }
+    end
+
+    @bucket.put_object(
+      key, file: '/tmp/x', content_type: 'text/plain')
+
+    @bucket.get_object(
+      key, file: '/tmp/y', headers: {'accept-encoding': 'deflate'})
+
+    assert File.exist?('/tmp/y')
+    diff = `diff /tmp/x /tmp/y`
+    assert diff.empty?
+  end
+end