Convert string to bytes in StreamWriter

lib/aliyun/oss/http.rb

@@ -41,18 +41,22 @@ class HTTP
       #
       class StreamWriter
         def initialize
-          @chunks = []
+          @buffer = ""
           @producer = Fiber.new { yield self if block_given? }
           @producer.resume
         end
 
-        # FIXME: it may return more than bytes, not sure if that's a problem
         def read(bytes = nil, outbuf = nil)
           ret = ""
           loop do
-            c = @chunks.shift
-            ret << c if c && !c.empty?
-            break if bytes && ret.size >= bytes
+            if bytes
+              ret << @buffer.slice!(0, bytes)
+              break if ret.size >= bytes
+            else
+              ret << @buffer
+              @buffer.clear
+            end
+
             if @producer.alive?
               @producer.resume
             else
@@ -76,7 +80,7 @@ def read(bytes = nil, outbuf = nil)
         end
 
         def write(chunk)
-          @chunks << chunk
+          @buffer << chunk.to_s.force_encoding(Encoding::ASCII_8BIT)
           Fiber.yield
           self
         end
@@ -88,7 +92,7 @@ def closed?
         end
 
         def inspect
-          "@chunks: " + @chunks.map { |c| c[0, 100].inspect }.join(';')
+          "@buffer: " + @buffer[0, 32].inspect + "...#{@buffer.size} bytes"
         end
       end
 

spec/aliyun/oss/http_spec.rb

@@ -10,14 +10,53 @@ module OSS
       context HTTP::StreamWriter do
         it "should read out chunks that are written" do
           s = HTTP::StreamWriter.new do |sr|
-            10.times{ |i| sr << "hello, #{i}" }
+            100.times { sr << "x" }
           end
 
-          10.times do |i|
-            bytes, outbuf = "hello, 0".size, ""
+          10.times do
+            bytes, outbuf = 10, ''
             s.read(bytes, outbuf)
-            expect(outbuf).to eq("hello, #{i}")
+            expect(outbuf).to eq("x"*10)
           end
+
+          outbuf = 'xxx'
+          r = s.read(10, outbuf)
+          expect(outbuf).to eq('')
+          expect(r).to be nil
+
+          r = s.read
+          expect(outbuf.empty?).to be true
+          expect(r).to eq('')
+        end
+
+        it "should convert chunk to string" do
+          s = HTTP::StreamWriter.new do |sr|
+            sr << 100 << 200
+          end
+
+          r = s.read
+          expect(r).to eq("100200")
+        end
+
+        it "should encode string to bytes" do
+          s = HTTP::StreamWriter.new do |sr|
+            100.times { sr << "中" }
+          end
+
+          r = s.read(1)
+          expect(r).to eq('中'.force_encoding(Encoding::ASCII_8BIT)[0])
+          s.read(2)
+          r = s.read(3)
+          expect(r.force_encoding(Encoding::UTF_8)).to eq('中')
+
+          bytes = (100 - 2) * 3
+          outbuf = 'zzz'
+          r = s.read(bytes, outbuf)
+          expect(outbuf.size).to eq(bytes)
+          expect(r.size).to eq(bytes)
+
+          r = s.read
+          expect(r).to eq('')
         end
       end # StreamWriter
 

tests/test_content_type.rb

@@ -11,8 +11,8 @@ def setup
     client = Aliyun::OSS::Client.new(
       :endpoint => conf['endpoint'],
       :cname => conf['cname'],
-      :access_key_id => conf['id'],
-      :access_key_secret => conf['key'])
+      :access_key_id => conf['access_key_id'],
+      :access_key_secret => conf['access_key_secret'])
     @bucket = client.get_bucket(conf['bucket'])
 
     @types = {

tests/test_encoding.rb

@@ -0,0 +1,81 @@
+# coding: utf-8
+require 'minitest/autorun'
+require 'yaml'
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+require 'aliyun/oss'
+
+class TestEncoding < Minitest::Test
+  def setup
+    Aliyun::OSS::Logging.set_log_level(Logger::DEBUG)
+    conf_file = '~/.oss.yml'
+    conf = YAML.load(File.read(File.expand_path(conf_file)))
+    client = Aliyun::OSS::Client.new(
+      :endpoint => conf['endpoint'],
+      :cname => conf['cname'],
+      :access_key_id => conf['access_key_id'],
+      :access_key_secret => conf['access_key_secret'])
+    @bucket = client.get_bucket(conf['bucket'])
+    @prefix = 'tests/encoding/'
+  end
+
+  def get_key(k)
+    @prefix + k
+  end
+
+  def test_utf_8
+    key = get_key('utf-8')
+    @bucket.put_object(key) do |stream|
+      stream << '中国' << 'Ruby'
+    end
+    put = '中国Ruby'.force_encoding(Encoding::ASCII_8BIT)
+    got = ''
+    @bucket.get_object(key) { |c| got << c }
+    assert_equal put, got
+
+    File.open('/tmp/x', 'w') { |f| f.write('中国Ruby') }
+    @bucket.put_object(key, :file => '/tmp/x')
+    got = ''
+    @bucket.get_object(key) { |c| got << c }
+    assert_equal put, got
+  end
+
+  def test_gbk
+    key = get_key('gbk')
+    @bucket.put_object(key) do |stream|
+      stream << '中国'.encode(Encoding::GBK) << 'Ruby'.encode(Encoding::GBK)
+    end
+    put = '中国Ruby'.encode(Encoding::GBK).force_encoding(Encoding::ASCII_8BIT)
+    got = ''
+    @bucket.get_object(key) { |c| got << c }
+    assert_equal put, got
+
+    File.open('/tmp/x', 'w') { |f| f.write('中国Ruby'.encode(Encoding::GBK)) }
+    @bucket.put_object(key, :file => '/tmp/x')
+    got = ''
+    @bucket.get_object(key) { |c| got << c }
+    assert_equal put, got
+  end
+
+  def encode_number(i)
+    [i].pack('N')
+  end
+
+  def test_binary
+    key = get_key('bin')
+    @bucket.put_object(key) do |stream|
+      (0..1024).each { |i| stream << encode_number(i) }
+    end
+    put = (0..1024).reduce('') { |s, i| s << encode_number(i) }
+    got = ''
+    @bucket.get_object(key) { |c| got << c }
+    assert_equal put, got
+
+    File.open('/tmp/x', 'w') { |f|
+      (0..1024).each { |i| f.write(encode_number(i)) }
+    }
+    @bucket.put_object(key, :file => '/tmp/x')
+    got = ''
+    @bucket.get_object(key) { |c| got << c }
+    assert_equal put, got
+  end
+end

tests/test_large_file.rb

@@ -12,8 +12,8 @@ def setup
     client = Aliyun::OSS::Client.new(
       :endpoint => conf['endpoint'],
       :cname => conf['cname'],
-      :access_key_id => conf['id'],
-      :access_key_secret => conf['key'])
+      :access_key_id => conf['access_key_id'],
+      :access_key_secret => conf['access_key_secret'])
     @bucket = client.get_bucket(conf['bucket'])
     @prefix = 'tests/large_file/'
   end

tests/test_multipart.rb

@@ -4,15 +4,15 @@
 $LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
 require 'aliyun/oss'
 
-class TestMultipart < Minitest::Unit::TestCase
+class TestMultipart < Minitest::Test
   def setup
     conf_file = '~/.oss.yml'
     conf = YAML.load(File.read(File.expand_path(conf_file)))
     opts = {
       endpoint: conf['endpoint'],
       cname: conf['cname'],
-      access_key_id: conf['id'],
-      access_key_secret: conf['key'],
+      access_key_id: conf['access_key_id'],
+      access_key_secret: conf['access_key_secret'],
     }
     client = Aliyun::OSS::Client.new(opts)
     @bucket_name = conf['bucket']

tests/test_object_key.rb

@@ -12,8 +12,8 @@ def setup
     client = Aliyun::OSS::Client.new(
       :endpoint => conf['endpoint'],
       :cname => conf['cname'],
-      :access_key_id => conf['id'],
-      :access_key_secret => conf['key'])
+      :access_key_id => conf['access_key_id'],
+      :access_key_secret => conf['access_key_secret'])
     @bucket = client.get_bucket(conf['bucket'])
     @prefix = 'tests/object_key/'
     @keys = {

tests/test_resumable.rb

@@ -11,8 +11,8 @@ def setup
     client = Aliyun::OSS::Client.new(
       :endpoint => conf['endpoint'],
       :cname => conf['cname'],
-      :access_key_id => conf['id'],
-      :access_key_secret => conf['key'])
+      :access_key_id => conf['access_key_id'],
+      :access_key_secret => conf['access_key_secret'])
     @bucket = client.get_bucket(conf['bucket'])
     @prefix = 'tests/resumable/'
   end