Merge 216ffaf into 81c9bc0

Rakefile

@@ -54,13 +54,13 @@ task :smart_test do
     begin
       env_crc_enable = ENV['RUBY_SDK_OSS_CRC_ENABLE']
 
-      # run test without crc
-      ENV['RUBY_SDK_OSS_CRC_ENABLE'] = nil if ENV['RUBY_SDK_OSS_CRC_ENABLE']
-      Rake::Task[:test].invoke
-
       # run test with crc
       ENV['RUBY_SDK_OSS_CRC_ENABLE'] = 'true'
       Rake::Task[:test].invoke
+
+      # run test without crc
+      ENV['RUBY_SDK_OSS_CRC_ENABLE'] = 'false'
+      Rake::Task[:test].invoke
     ensure
       ENV['RUBY_SDK_OSS_CRC_ENABLE'] = env_crc_enable
     end

lib/aliyun/oss/client.rb

@@ -28,6 +28,8 @@ class Client
       #  KEY SECRET，如果不填则会尝试匿名访问
       # @option opts [Boolean] :cname [可选] 指定endpoint是否是用户绑
       #  定的域名
+      # @option opts [Boolean] :crc_enable [可选]指定是否开启CRC校验，
+      #  默认为开启(true)
       # @option opts [String] :sts_token [可选] 指定STS的
       #  SecurityToken，如果指定，则使用STS授权访问
       # @option opts [Fixnum] :open_timeout [可选] 指定建立连接的超时

lib/aliyun/oss/config.rb

@@ -20,7 +20,7 @@ def initialize(opts = {})
         @access_key_id = @access_key_id.strip if @access_key_id
         @access_key_secret = @access_key_secret.strip if @access_key_secret
         normalize_endpoint if endpoint
-        @crc_enable ||= false
+        @crc_enable = (@crc_enable == 'false' || @crc_enable == false) ? false : true
       end
 
       private

lib/aliyun/oss/protocol.rb

@@ -546,7 +546,7 @@ def put_object(bucket_name, object_name, opts = {}, &block)
           raise e
         end
 
-        if @config.crc_enable 
+        if @config.crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
           data_crc = payload.read.data_crc
           Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
         end
@@ -599,7 +599,9 @@ def append_object(bucket_name, object_name, position, opts = {}, &block)
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
           {:headers => headers, :body => payload})
 
-        if @config.crc_enable && !opts[:init_crc].nil?
+        if @config.crc_enable && 
+          !r.headers[:x_oss_hash_crc64ecma].nil? && 
+          !opts[:init_crc].nil?
           data_crc = payload.read.data_crc
           Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'append')
         end
@@ -773,11 +775,22 @@ def get_object(bucket_name, object_name, opts = {}, &block)
             rewrites[:expires].httpdate if rewrites.key?(:expires)
         end
 
+        data_crc = opts[:init_crc].nil? ? 0 : opts[:init_crc]
         r = @http.get(
           {:bucket => bucket_name, :object => object_name,
            :sub_res => sub_res},
           {:headers => headers}
-        ) { |chunk| yield chunk if block_given? }
+        ) do |chunk| 
+          if block_given?
+            # crc enable and no range and oss server support crc
+            data_crc = Aliyun::OSS::Util.crc(chunk, data_crc) if @config.crc_enable && range.nil?
+            yield chunk
+          end
+        end
+
+        if @config.crc_enable && range.nil? && !r.headers[:x_oss_hash_crc64ecma].nil?
+          Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'get')
+        end
 
         h = r.headers
         metas = {}
@@ -1102,7 +1115,7 @@ def upload_part(bucket_name, object_name, txn_id, part_no, &block)
           {:bucket => bucket_name, :object => object_name, :sub_res => sub_res},
           {:body => payload})
 
-        if @config.crc_enable
+        if @config.crc_enable && !r.headers[:x_oss_hash_crc64ecma].nil?
           data_crc = payload.read.data_crc
           Aliyun::OSS::Util.crc_check(data_crc, r.headers[:x_oss_hash_crc64ecma], 'put')
         end
@@ -1400,7 +1413,7 @@ def sign(string_to_sign)
       end
 
       # Get the crc status
-      # @return true(crc enable) or false(crc disable)
+      # @return [Boolean] true(crc enable) or false(crc disable)
       def crc_enable
         @config.crc_enable
       end

spec/aliyun/oss/object_spec.rb

@@ -661,6 +661,55 @@ def err(msg, reqid = '0000')
           expect(WebMock).to have_requested(:get, url)
             .with(:body => nil, :query => query)
         end
+
+
+        it "should raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
+          response_content = ""
+          expect(crc_protocol.crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name) {|c| response_content << c}
+          }.to raise_error(CrcInconsistentError, "The crc of get between client and oss is not inconsistent.")
+        end
+
+        it "should not raise crc exception on error" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc})
+          response_content = ""
+          expect(crc_protocol.crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name) {|c| response_content << c}
+          }.not_to raise_error
+          expect(response_content).to eq(content)
+        end
+
+        it "should not raise crc exception on error when setting range" do
+          object_name = 'ruby'
+          url = get_request_path(object_name)
+          content = "hello world 0123456789"
+          content_crc = Aliyun::OSS::Util.crc(content)
+          stub_request(:get, url).to_return(
+            :status => 200, :body => content, :headers => {:x_oss_hash_crc64ecma => content_crc.to_i + 1})
+          response_content = ""
+          expect(crc_protocol.crc_enable).to eq(true)
+          expect {
+            crc_protocol.get_object(@bucket, object_name, {range: [0, 10]}) {|c| response_content << c}
+          }.not_to raise_error
+          expect(WebMock).to have_requested(:get, url)
+            .with(:body => nil, :query => {},
+                  :headers => {
+                    'Range' => 'bytes=0-9'
+                  })
+        end
       end # Get object
 
       context "Get object meta" do

tests/helper.rb

@@ -0,0 +1,15 @@
+# -*- encoding: utf-8 -*-
+
+module Aliyun
+  module Test
+    module Helper
+      def random_string(n)
+        (1...n).map { (65 + rand(26)).chr }.join + "\n"
+      end
+
+      def random_bytes(n)
+        (1...n).map { rand(255).chr }.join + "\n"
+      end
+    end
+  end
+end

tests/test_crc_check.rb

@@ -0,0 +1,184 @@
+require 'minitest/autorun'
+require 'yaml'
+require 'tempfile'
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+require 'aliyun/oss'
+require_relative 'config'
+require_relative 'helper'
+
+class TestCrcCheck < Minitest::Test
+
+  include Aliyun::Test::Helper
+
+  @@tests_run = 0
+  @@test_file = nil
+
+  def setup
+    Aliyun::Common::Logging.set_log_level(Logger::DEBUG)
+    client = Aliyun::OSS::Client.new(TestConf.creds)
+    @bucket = client.get_bucket(TestConf.bucket)
+    @prefix = 'tests/crc_check/'
+
+    if @@tests_run == 0
+      @@test_file = Tempfile.new('oss_ruby_sdk_test_crc')
+      (10 * 1024).times { @@test_file.write(random_bytes(1024)) }
+    end
+    @@tests_run += 1
+  end
+
+  def teardown
+    if @@tests_run == TestCrcCheck.runnable_methods.length
+      @@test_file.unlink unless @@test_file.nil?
+    end
+  end
+
+  def get_key(k)
+    @prefix + k
+  end
+
+  def test_put_object
+    skip unless TestConf.creds[:crc_enable]
+
+    # Check crc status
+    assert(@bucket.crc_enable)
+
+    # Create a test file with 10MB random bytes to put.
+    key = get_key('put_file')
+
+    @bucket.put_object(key, :file => @@test_file.path)
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 10 * 1024 * 1024)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.put_object(key, {:init_crc => 1, :file => @@test_file.path}) do |content|
+        content << 'hello world.'
+      end
+    end
+
+    # Put a string to oss.
+    key = get_key('put_string')
+    @bucket.put_object(key, :init_crc => 0) do |content|
+      content << 'hello world.'
+    end
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.put_object(key, :init_crc => 1) do |content|
+        content << 'hello world.'
+      end
+    end
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_append_object
+    skip unless TestConf.creds[:crc_enable]
+    key = get_key('append_file')
+
+    # Check crc status
+    assert(@bucket.crc_enable)
+
+    # Check $key object doesn't exist.
+    test_object = @bucket.get_object(key) rescue 0
+    @bucket.delete_object(key) if test_object.size
+
+    # Create a append object to oss with a string.
+    position = @bucket.append_object(key, 0, :init_crc => 0) do |content|
+      content << 'hello world.'
+    end
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size)
+
+    # Append a test file to oss $key object.
+    @bucket.append_object(key, position, {:init_crc => test_object.headers[:x_oss_hash_crc64ecma], :file => @@test_file.path})
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size + (10 * 1024 * 1024))
+
+    # No crc check when init_crc is nil
+    position = @bucket.append_object(key, test_object.size) do |content|
+      content << 'hello world.'
+    end
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size * 2 + (10 * 1024 * 1024))
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      position = @bucket.append_object(key, test_object.size, :init_crc => 0) do |content|
+        content << 'hello world.'
+      end
+    end
+
+    # Check crc wrong case.
+    test_object = @bucket.get_object(key)
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.append_object(key, test_object.size, {:init_crc => 0, :file => @@test_file.path})
+    end
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_upload_object
+    skip unless TestConf.creds[:crc_enable]
+    key = get_key('upload_file')
+
+    # Check crc status
+    assert(@bucket.crc_enable)
+    @bucket.resumable_upload(key, @@test_file.path, :cpt_file => "#{@@test_file.path}.cpt", threads: 2, :part_size => 1024 * 1024)
+
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, (10 * 1024 * 1024))
+
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_get_small_object
+    skip unless TestConf.creds[:crc_enable]
+
+    # Check crc status
+    assert(@bucket.crc_enable)
+
+    # Put a string to oss.
+    key = get_key('get_small_object')
+    @bucket.put_object(key) do |content|
+      content << 'hello world.'
+    end
+    temp_buf = ""
+    test_object = @bucket.get_object(key) { |c| temp_buf << c }
+    assert_equal(test_object.size, 'hello world.'.size)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.get_object(key, {:init_crc => 1}) { |c| temp_buf << c }
+    end
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_get_large_object
+    skip unless TestConf.creds[:crc_enable]
+
+    # Check crc status
+    assert(@bucket.crc_enable)
+
+    # put a test file with 10MB random bytes to oss for testing get.
+    key = get_key('get_file')
+    @bucket.put_object(key, :file => @@test_file.path)
+
+    get_temp_file = Tempfile.new('oss_ruby_sdk_test_crc_get')
+    test_object = @bucket.get_object(key, {:file => get_temp_file})
+    assert_equal(test_object.size, 10 * 1024 * 1024)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.get_object(key, {:file => get_temp_file, :init_crc => 1})
+    end
+  ensure
+    get_temp_file.unlink unless get_temp_file.nil?
+    @bucket.delete_object(key)
+  end
+
+end