Merge dee3b05 into 81c9bc0

Rakefile

@@ -52,17 +52,21 @@ task :smart_test do
 
   if ENV.keys.include?('RUBY_SDK_OSS_KEY')
     begin
-      env_crc_enable = ENV['RUBY_SDK_OSS_CRC_ENABLE']
+      env_upload_crc_enable = ENV['RUBY_SDK_OSS_UPLOAD_CRC_ENABLE']
+      env_download_crc_enable = ENV['RUBY_SDK_OSS_DOWNLOAD_CRC_ENABLE']
 
-      # run test without crc
-      ENV['RUBY_SDK_OSS_CRC_ENABLE'] = nil if ENV['RUBY_SDK_OSS_CRC_ENABLE']
+      # run test with crc
+      ENV['RUBY_SDK_OSS_UPLOAD_CRC_ENABLE'] = 'true'
+      ENV['RUBY_SDK_OSS_DOWNLOAD_CRC_ENABLE'] = 'true'
       Rake::Task[:test].invoke
 
-      # run test with crc
-      ENV['RUBY_SDK_OSS_CRC_ENABLE'] = 'true'
+      # run test without crc
+      ENV['RUBY_SDK_OSS_UPLOAD_CRC_ENABLE'] = 'false'
+      ENV['RUBY_SDK_OSS_DOWNLOAD_CRC_ENABLE'] = 'false'
       Rake::Task[:test].invoke
     ensure
-      ENV['RUBY_SDK_OSS_CRC_ENABLE'] = env_crc_enable
+      ENV['RUBY_SDK_OSS_UPLOAD_CRC_ENABLE'] = env_upload_crc_enable
+      ENV['RUBY_SDK_OSS_DOWNLOAD_CRC_ENABLE'] = env_download_crc_enable
     end
   end
 end

lib/aliyun/oss/bucket.rb

@@ -639,10 +639,16 @@ def sign(string_to_sign)
         @protocol.sign(string_to_sign)
       end
 
-      # Get the crc status
-      # @return true(crc enable) or false(crc disable)
-      def crc_enable
-        @protocol.crc_enable
+      # Get the download crc status
+      # @return true(download crc enable) or false(download crc disable)
+      def download_crc_enable
+        @protocol.download_crc_enable
+      end
+
+      # Get the upload crc status
+      # @return true(upload crc enable) or false(upload crc disable)
+      def upload_crc_enable
+        @protocol.upload_crc_enable
       end
 
       private

lib/aliyun/oss/client.rb

@@ -28,6 +28,10 @@ class Client
       #  KEY SECRET，如果不填则会尝试匿名访问
       # @option opts [Boolean] :cname [可选] 指定endpoint是否是用户绑
       #  定的域名
+      # @option opts [Boolean] :upload_crc_enable [可选]指定上传处理
+      #  是否开启CRC校验，默认为开启(true)
+      # @option opts [Boolean] :download_crc_enable [可选]指定下载处理
+      #  是否开启CRC校验，默认为不开启(false)
       # @option opts [String] :sts_token [可选] 指定STS的
       #  SecurityToken，如果指定，则使用STS授权访问
       # @option opts [Fixnum] :open_timeout [可选] 指定建立连接的超时

lib/aliyun/oss/config.rb

@@ -12,15 +12,16 @@ class Config < Common::Struct::Base
       attrs :endpoint, :cname, :sts_token,
             :access_key_id, :access_key_secret,
             :open_timeout, :read_timeout,
-            :crc_enable
+            :download_crc_enable, :upload_crc_enable
 
       def initialize(opts = {})
         super(opts)
 
         @access_key_id = @access_key_id.strip if @access_key_id
         @access_key_secret = @access_key_secret.strip if @access_key_secret
         normalize_endpoint if endpoint
-        @crc_enable ||= false
+        @upload_crc_enable = (@upload_crc_enable == 'false' || @upload_crc_enable == false) ? false : true
+        @download_crc_enable = (@download_crc_enable == 'true' || @download_crc_enable == true) ? true : false
       end
 
       private

lib/aliyun/oss/protocol.rb

@@ -535,7 +535,7 @@ def put_object(bucket_name, object_name, opts = {}, &block)
           headers[CALLBACK_HEADER] = opts[:callback].serialize
         end
 
-        payload = HTTP::StreamPayload.new(@config.crc_enable, opts[:init_crc], &block)
+        payload = HTTP::StreamPayload.new(@config.upload_crc_enable, opts[:init_crc], &block)
         r = @http.put(
           {:bucket => bucket_name, :object => object_name},
           {:headers => headers, :body => payload})
@@ -546,7 +546,7 @@ def put_object(bucket_name, object_name, opts = {}, &block)
           raise e
         end
 
-        if @config.crc_enable 
+        if @config.upload_crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
           data_crc = payload.read.data_crc
           Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
         end
@@ -593,13 +593,15 @@ def append_object(bucket_name, object_name, position, opts = {}, &block)
 
         headers.merge!(to_lower_case(opts[:headers])) if opts.key?(:headers)
 
-        payload = HTTP::StreamPayload.new(@config.crc_enable && !opts[:init_crc].nil?, opts[:init_crc], &block)
+        payload = HTTP::StreamPayload.new(@config.upload_crc_enable && !opts[:init_crc].nil?, opts[:init_crc], &block)
 
         r = @http.post(
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
           {:headers => headers, :body => payload})
 
-        if @config.crc_enable && !opts[:init_crc].nil?
+        if @config.upload_crc_enable && 
+          !r.headers[:x_oss_hash_crc64ecma].nil? && 
+          !opts[:init_crc].nil?
           data_crc = payload.read.data_crc
           Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'append')
         end
@@ -773,11 +775,22 @@ def get_object(bucket_name, object_name, opts = {}, &block)
             rewrites[:expires].httpdate if rewrites.key?(:expires)
         end
 
+        data_crc = opts[:init_crc].nil? ? 0 : opts[:init_crc]
         r = @http.get(
           {:bucket => bucket_name, :object => object_name,
            :sub_res => sub_res},
           {:headers => headers}
-        ) { |chunk| yield chunk if block_given? }
+        ) do |chunk| 
+          if block_given?
+            # crc enable and no range and oss server support crc
+            data_crc = Aliyun::OSS::Util.crc(chunk, data_crc) if @config.download_crc_enable && range.nil?
+            yield chunk
+          end
+        end
+
+        if @config.download_crc_enable && range.nil? && !r.headers[:x_oss_hash_crc64ecma].nil?
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'get')
+        end
 
         h = r.headers
         metas = {}
@@ -1097,12 +1110,12 @@ def upload_part(bucket_name, object_name, txn_id, part_no, &block)
 
         sub_res = {'partNumber' => part_no, 'uploadId' => txn_id}
 
-        payload = HTTP::StreamPayload.new(@config.crc_enable, &block)
+        payload = HTTP::StreamPayload.new(@config.upload_crc_enable, &block)
         r = @http.put(
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
           {:body => payload})
 
-        if @config.crc_enable
+        if @config.upload_crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
           data_crc = payload.read.data_crc
           Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
         end
@@ -1398,11 +1411,17 @@ def get_sts_token
       def sign(string_to_sign)
         Util.sign(@config.access_key_secret, string_to_sign)
       end
-
-      # Get the crc status
-      # @return true(crc enable) or false(crc disable)
-      def crc_enable
-        @config.crc_enable
+
+      # Get the download crc status
+      # @return true(download crc enable) or false(download crc disable)
+      def download_crc_enable
+        @config.download_crc_enable
+      end
+
+      # Get the upload crc status
+      # @return true(upload crc enable) or false(upload crc disable)
+      def upload_crc_enable
+        @config.upload_crc_enable
       end
 
       private

spec/aliyun/oss/multipart_spec.rb

@@ -28,7 +28,8 @@ def crc_protocol
           Config.new(:endpoint => @endpoint,
                      :access_key_id => 'xxx', 
                      :access_key_secret => 'yyy', 
-                     :crc_enable => true))
+                     :upload_crc_enable => true,
+                     :download_crc_enable => true))
       end
 
       def mock_txn_id(txn_id)

spec/aliyun/oss/object_spec.rb

@@ -22,7 +22,8 @@ def crc_protocol
           Config.new(:endpoint => @endpoint,
                      :access_key_id => 'xxx', 
                      :access_key_secret => 'yyy', 
-                     :crc_enable => true))
+                     :upload_crc_enable => true,
+                     :download_crc_enable => true))
       end
 
       def get_request_path(object = nil)
@@ -205,8 +206,7 @@ def err(msg, reqid = '0000')
           content_crc = Aliyun::OSS::Util.crc(content)
           stub_request(:put, url).to_return(
             :status => 200, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
-
-          expect(crc_protocol.crc_enable).to eq(true)
+          expect(crc_protocol.upload_crc_enable).to eq(true)
           expect {
             crc_protocol.put_object(@bucket, object_name) do |c|
               c << content
@@ -221,7 +221,7 @@ def err(msg, reqid = '0000')
           content_crc = Aliyun::OSS::Util.crc(content)
           stub_request(:put, url).to_return(
             :status => 200, :headers => {:x_oss_hash_crc64ecma => content_crc})
-          expect(crc_protocol.crc_enable).to eq(true)
+          expect(crc_protocol.upload_crc_enable).to eq(true)
           expect {
             crc_protocol.put_object(@bucket, object_name) do |c|
               c << content
@@ -339,7 +339,7 @@ def err(msg, reqid = '0000')
           return_headers = {'x-oss-next-append-position' => '101', :x_oss_hash_crc64ecma => content_crc.to_i + 1}
           stub_request(:post, url).with(:query => query)
             .to_return(:headers => return_headers)
-          expect(crc_protocol.crc_enable).to eq(true)
+          expect(crc_protocol.upload_crc_enable).to eq(true)
           expect {
             crc_protocol.append_object(@bucket, object_name, 11, :init_crc => 0) do |c|
               c << content
@@ -358,7 +358,7 @@ def err(msg, reqid = '0000')
           stub_request(:post, url).with(:query => query)
             .to_return(:headers => return_headers)
 
-          expect(crc_protocol.crc_enable).to eq(true)
+          expect(crc_protocol.upload_crc_enable).to eq(true)
           next_pos = 0
           expect {
             next_pos = crc_protocol.append_object(@bucket, object_name, 11, :init_crc => 0) do |c|
@@ -382,7 +382,7 @@ def err(msg, reqid = '0000')
           stub_request(:post, url).with(:query => query)
             .to_return(:headers => return_headers)
 
-          expect(crc_protocol.crc_enable).to eq(true)
+          expect(crc_protocol.upload_crc_enable).to eq(true)
           next_pos = 0
           expect {
             next_pos = crc_protocol.append_object(@bucket, object_name, 11) do |c|
@@ -661,6 +661,55 @@ def err(msg, reqid = '0000')
           expect(WebMock).to have_requested(:get, url)
             .with(:body => nil, :query => query)
         end
+
+
+        it "should raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
+          response_content = ""
+          expect(crc_protocol.download_crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name) {|c| response_content << c}
+          }.to raise_error(CrcInconsistentError, "The crc of get between client and oss is not inconsistent.")
+        end
+
+        it "should not raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc})
+          response_content = ""
+          expect(crc_protocol.download_crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name) {|c| response_content << c}
+          }.not_to raise_error
+          expect(response_content).to eq(content)
+        end
+
+        it "should not raise crc exception on error when setting range" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world 0123456789"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
+          response_content = ""
+          expect(crc_protocol.download_crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name, {range: [0, 10]}) {|c| response_content << c}
+          }.not_to raise_error
+          expect(WebMock).to have_requested(:get, url)
+            .with(:body => nil, :query => {},
+                  :headers => {
+                    'Range' => 'bytes=0-9'
+                  })
+        end
       end # Get object
 
       context "Get object meta" do

tests/config.rb

@@ -4,7 +4,8 @@ def creds
       {
         access_key_id: ENV['RUBY_SDK_OSS_ID'],
         access_key_secret: ENV['RUBY_SDK_OSS_KEY'],
-        crc_enable: ENV['RUBY_SDK_OSS_CRC_ENABLE'],
+        download_crc_enable: ENV['RUBY_SDK_OSS_DOWNLOAD_CRC_ENABLE'],
+        upload_crc_enable: ENV['RUBY_SDK_OSS_UPLOAD_CRC_ENABLE'],
         endpoint: ENV['RUBY_SDK_OSS_ENDPOINT']
       }
     end

tests/helper.rb

@@ -0,0 +1,15 @@
+# -*- encoding: utf-8 -*-
+
+module Aliyun
+  module Test
+    module Helper
+      def random_string(n)
+        (1...n).map { (65 + rand(26)).chr }.join + "\n"
+      end
+
+      def random_bytes(n)
+        (1...n).map { rand(255).chr }.join + "\n"
+      end
+    end
+  end
+end