added streaming support [#17]

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    s3 (0.3.3)
+    s3 (0.3.4)
       proxies
 
 GEM

Rakefile

@@ -1,5 +1,6 @@
 require "bundler"
 Bundler::GemHelper.install_tasks
+Bundler.setup
 
 require "rake/testtask"
 require "rake/rdoctask"

lib/s3.rb

@@ -15,6 +15,7 @@
 require "s3/bucket"
 require "s3/connection"
 require "s3/exceptions"
+require "s3/request"
 require "s3/object"
 require "s3/service"
 require "s3/signature"

lib/s3/connection.rb

@@ -19,14 +19,18 @@ class Connection
     # * <tt>:timeout</tt> - Timeout to use by the Net::HTTP object
     #   (60 by default)
     # * <tt>:proxy</tt> - Hash for Net::HTTP Proxy settings
-    # { :host => "proxy.mydomain.com", :port => "80, :user => "user_a", :password => "secret" }
+    #   { :host => "proxy.mydomain.com", :port => "80, :user => "user_a", :password => "secret" }
+    # * <tt>:proxy</tt> - Hash for Net::HTTP Proxy settings
+    # * <tt>:chunk_size</tt> - Size of a chunk when streaming
+    #   (1048576 (1 MiB) by default)
     def initialize(options = {})
       @access_key_id = options.fetch(:access_key_id)
       @secret_access_key = options.fetch(:secret_access_key)
       @use_ssl = options.fetch(:use_ssl, false)
       @debug = options.fetch(:debug, false)
       @timeout = options.fetch(:timeout, 60)
       @proxy = options.fetch(:proxy, nil)
+      @chunk_size = options.fetch(:chunk_size, 1048576)
     end
 
     # Makes request with given HTTP method, sets missing parameters,
@@ -53,7 +57,7 @@ def initialize(options = {})
     def request(method, options)
       host = options.fetch(:host, HOST)
       path = options.fetch(:path)
-      body = options.fetch(:body, "")
+      body = options.fetch(:body, nil)
       params = options.fetch(:params, {})
       headers = options.fetch(:headers, {})
 
@@ -63,14 +67,21 @@ def request(method, options)
       end
 
       path = URI.escape(path)
-      request = request_class(method).new(path)
+      request = Request.new(@chunk_size, method.to_s.upcase, !!body, true, path)
 
       headers = self.class.parse_headers(headers)
       headers.each do |key, value|
         request[key] = value
       end
 
-      request.body = body
+      if body
+        if body.respond_to?(:read)
+          request.body_stream = body
+        else
+          request.body = body
+        end
+        request.content_length = body.respond_to?(:lstat) ? body.stat.size : body.size
+      end
 
       send_request(host, request)
     end
@@ -143,19 +154,6 @@ def self.parse_headers(headers)
 
     private
 
-    def request_class(method)
-      case method
-      when :get
-        request_class = Net::HTTP::Get
-      when :head
-        request_class = Net::HTTP::Head
-      when :put
-        request_class = Net::HTTP::Put
-      when :delete
-        request_class = Net::HTTP::Delete
-      end
-    end
-
     def port
       use_ssl ? 443 : 80
     end

lib/s3/object.rb

@@ -40,8 +40,8 @@ def key=(key)
     def acl=(acl)
       @acl = acl.to_s.gsub("_", "-") if acl
     end
-    
-    # Assigns a new storage class (RRS) to the object. Please note 
+
+    # Assigns a new storage class (RRS) to the object. Please note
     # that the storage class is not retrieved from the server and set
     # to "STANDARD" by default.
     #
@@ -185,8 +185,7 @@ def object_headers(options = {})
     end
 
     def put_object
-      body = content.is_a?(IO) ? content.read : content
-      response = object_request(:put, :body => body, :headers => dump_headers)
+      response = object_request(:put, :body => content, :headers => dump_headers)
       parse_headers(response)
     end
 

lib/s3/request.rb

@@ -0,0 +1,31 @@
+module S3
+  # Class responsible for sending chunked requests
+  # properly. Net::HTTPGenericRequest has hardcoded chunk_size, so we
+  # inherit the class and override chunk_size.
+  class Request < Net::HTTPGenericRequest
+    def initialize(chunk_size, m, reqbody, resbody, path, initheader = nil)
+      @chunk_size = chunk_size
+      super(m, reqbody, resbody, path, initheader)
+    end
+
+    private
+
+    def send_request_with_body_stream(sock, ver, path, f)
+      unless content_length() or chunked?
+        raise ArgumentError, "Content-Length not given and Transfer-Encoding is not `chunked'"
+      end
+      supply_default_content_type
+      write_header sock, ver, path
+      if chunked?
+        while s = f.read(@chunk_size)
+          sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
+        end
+        sock.write "0\r\n\r\n"
+      else
+        while s = f.read(@chunk_size)
+          sock.write s
+        end
+      end
+    end
+  end
+end

lib/s3/service.rb

@@ -22,12 +22,15 @@ def ==(other)
     #   (false by default)
     # * <tt>:timeout</tt> - Timeout to use by the Net::HTTP object
     #   (60 by default)
+    # * <tt>:chunk_size</tt> - Size of a chunk when streaming
+    #   (1048576 (1 MiB) by default)
     def initialize(options)
       @access_key_id = options.fetch(:access_key_id)
       @secret_access_key = options.fetch(:secret_access_key)
       @use_ssl = options.fetch(:use_ssl, false)
       @timeout = options.fetch(:timeout, 60)
       @debug = options.fetch(:debug, false)
+      @chunk_size = options.fetch(:chunk_size, 1048576)
 
       raise ArgumentError, "Missing proxy settings. Must specify at least :host." if options[:proxy] && !options[:proxy][:host]
       @proxy = options.fetch(:proxy, nil)
@@ -74,7 +77,8 @@ def connection
                                      :use_ssl => @use_ssl,
                                      :timeout => @timeout,
                                      :debug => @debug,
-                                     :proxy => @proxy)
+                                     :proxy => @proxy,
+                                     :chunk_size => @chunk_size)
       end
       @connection
     end

lib/s3/version.rb

@@ -1,3 +1,3 @@
 module S3
-  VERSION = "0.3.4"
+  VERSION = "0.3.5"
 end

test/test_helper.rb

@@ -1,6 +1,3 @@
-require "rubygems"
-require "bundler/setup"
-
-Bundler.require
 require "test/unit"
 require "mocha"
+require "s3"