initial streaming support

spec/integration/crest_spec.cr

@@ -6,17 +6,6 @@ describe Crest do
     (response.body).should eq("Hello World!")
   end
 
-  it "do GET request with block" do
-    body = ""
-
-    response = Crest.get("#{TEST_SERVER_URL}/") do |resp|
-      body = resp.body
-    end
-
-    (response.body).should eq("Hello World!")
-    body.should eq("Hello World!")
-  end
-
   it "do GET request with params" do
     response = Crest.get("#{TEST_SERVER_URL}/resize", params: {:width => 100, :height => 100})
     (response.body).should eq("Width: 100, height: 100")
@@ -66,7 +55,7 @@ describe Crest do
   it "do GET request with block without handle errors" do
     body = ""
 
-    response = Crest.get("#{TEST_SERVER_URL}/404", handle_errors: false) do |resp|
+    Crest.get("#{TEST_SERVER_URL}/404", handle_errors: false) do |resp|
       case
       when resp.successful?
         body = resp.body
@@ -75,7 +64,6 @@ describe Crest do
       end
     end
 
-    puts response.status_code
     body.should eq("Not found.")
   end
 end

spec/integration/request_spec.cr

@@ -20,16 +20,6 @@ describe Crest::Request do
     (response.body).should eq("Post 1: comments")
   end
 
-  it "call get method with response block" do
-    body = ""
-    response = Crest::Request.get("#{TEST_SERVER_URL}/") do |resp|
-      body = resp.body
-    end
-
-    body.should eq("Hello World!")
-    (response.body).should eq("Hello World!")
-  end
-
   it "do GET request with params" do
     response = Crest::Request.execute(:get,
       "#{TEST_SERVER_URL}/resize",

spec/integration/streaming_spec.cr

@@ -0,0 +1,76 @@
+require "../spec_helper"
+
+describe Crest do
+  describe "Streaming" do
+    it "should stream Response#execute" do
+      body = String::Builder.new
+      request = Crest::Request.new(:get, "#{TEST_SERVER_URL}/")
+
+      request.execute do |resp|
+        while line = resp.body_io.gets
+          body << line
+        end
+      end
+
+      body.to_s.should eq("Hello World!")
+    end
+
+    it "should stream Request.execute" do
+      body = String::Builder.new
+      Crest::Request.get("#{TEST_SERVER_URL}/") do |resp|
+        while line = resp.body_io.gets
+          body << line
+        end
+      end
+
+      body.to_s.should eq("Hello World!")
+    end
+
+    it "should stream Request.get" do
+      body = String::Builder.new
+      Crest::Request.execute(:get, "#{TEST_SERVER_URL}/") do |resp|
+        while line = resp.body_io.gets
+          body << line
+        end
+      end
+
+      body.to_s.should eq("Hello World!")
+    end
+
+    it "should stream Crest#get" do
+      body = String::Builder.new
+      Crest.get("#{TEST_SERVER_URL}/") do |resp|
+        while line = resp.body_io.gets
+          body << line
+        end
+      end
+
+      body.to_s.should eq("Hello World!")
+    end
+
+    it "should stream Crest#get with redirects" do
+      body = String::Builder.new
+
+      Crest.get("#{TEST_SERVER_URL}/redirect/2") do |resp|
+        while line = resp.body_io.gets
+          body << line
+        end
+      end
+
+      body.to_s.should eq("Hello World!")
+    end
+
+    it "should stream Response#execute with redirects" do
+      body = String::Builder.new
+      request = Crest::Request.new(:get, "#{TEST_SERVER_URL}/redirect/2")
+
+      request.execute do |resp|
+        while line = resp.body_io.gets
+          body << line
+        end
+      end
+
+      body.to_s.should eq("Hello World!")
+    end
+  end
+end

src/crest.cr

@@ -24,7 +24,9 @@ require "./http/proxy/client"
 # )
 #
 # Crest.get("http://example.com/resource") do |response|
-#   File.write("file.html", response.body)
+#   while line = response.body_io.gets
+#     puts line
+#   end
 # end
 # ```
 module Crest
@@ -47,17 +49,14 @@ module Crest
     #
     # ```crystal
     # Crest.{{method.id}}("http://www.example.com") do |response|
-    #   File.write("file.html", response.body)
+    #   while line = response.body_io.gets
+    #     puts line
+    #   end
     # end
     # ```
-    def self.{{method.id}}(url : String, **args) : Crest::Response
+    def self.{{method.id}}(url : String, **args, &block : Crest::Response ->) : Nil
       request = Request.new(:{{method.id}}, url, **args)
-
-      response = exec(request)
-
-      yield response
-
-      response
+      request.execute(&block)
     end
 
     # Execute a {{method.id.upcase}} request and returns a `Crest::Response`.
@@ -66,15 +65,10 @@ module Crest
     # Crest.{{method.id}}("http://www.example.com")
     # ```
     def self.{{method.id}}(url : String, **args) : Crest::Response
-      {{method.id}}(url, **args) { }
+      request = Request.new(:{{method.id}}, url, **args)
+      request.execute
     end
-
   {% end %}
-
-  # Executes a `request`.
-  private def self.exec(request : Crest::Request) : Crest::Response
-    request.execute
-  end
 end
 
 require "./crest/**"

src/crest/redirector.cr

@@ -1,16 +1,13 @@
 module Crest
   class Redirector
-    def initialize(
-      @response : Crest::Response,
-      @request : Crest::Request
-    )
+    def initialize(@response : Crest::Response, @request : Crest::Request)
     end
 
     def follow : Crest::Response
-      case @response.status_code
-      when 200..207
+      case
+      when @response.successful?
         @response
-      when 301, 302, 303, 307
+      when @response.redirect?
         check_max_redirects
         follow_redirection
       else
@@ -19,6 +16,19 @@ module Crest
       end
     end
 
+    def follow(&block : Crest::Response ->)
+      case
+      when @response.successful?
+        @response
+      when @response.redirect?
+        check_max_redirects
+        follow_redirection(&block)
+      else
+        raise_exception! if @request.handle_errors
+        @response
+      end
+    end
+
     private def check_max_redirects
       raise_exception! if @request.max_redirects <= 0
     end
@@ -33,6 +43,14 @@ module Crest
       new_request.execute
     end
 
+    private def follow_redirection(&block : Crest::Response ->)
+      url = extract_url_from_headers
+
+      new_request = prepare_new_request(url)
+      new_request.redirection_history = @response.history + [@response]
+      new_request.execute(&block)
+    end
+
     private def extract_url_from_headers
       location_url = @response.headers["Location"].to_s
       location_uri = URI.parse(location_url)

src/crest/request.cr

@@ -74,11 +74,16 @@ module Crest
 
     delegate host, port, tls?, to: @http_client
 
-    def self.execute(method, url, **args)
+    def self.execute(method, url, **args) : Crest::Response
       request = new(method, url, **args)
       request.execute
     end
 
+    def self.execute(method, url, **args, &block : Crest::Response ->) : Nil
+      request = new(method, url, **args)
+      request.execute(&block)
+    end
+
     def initialize(
       method : Symbol,
       url : String,
@@ -137,19 +142,16 @@ module Crest
       #
       # ```crystal
       # Crest::Request.{{method.id}}("http://www.example.com") do |resp|
-      #   File.write("file.html", resp.body)
+      #   while line = resp.body_io.gets
+      #     puts line
+      #   end
       # end
       # ```
-      def self.{{method.id}}(url : String, **args) : Crest::Response
+      def self.{{method.id}}(url : String, **args, &block : Crest::Response ->) : Nil
         request = Request.new(:{{method.id}}, url, **args)
-
         request.basic_auth!(request.user, request.password)
 
-        response = request.execute
-
-        yield response
-
-        response
+        response = request.execute(&block)
       end
 
       # Execute a {{method.id.upcase}} request and returns a `Crest::Response`.
@@ -158,19 +160,51 @@ module Crest
       # Crest::Request.{{method.id}}("http://www.example.com")
       # ```
       def self.{{method.id}}(url : String, **args) : Crest::Response
-        {{method.id}}(url, **args) { }
+        request = Request.new(:{{method.id}}, url, **args)
+        request.basic_auth!(request.user, request.password)
+
+        request.execute
       end
     {% end %}
 
+    # Execute HTTP request
     def execute : Crest::Response
       @http_client.set_proxy(@proxy)
       @logger.request(self) if @logging
 
       @http_request = new_http_request(@method, @url, @headers, @form_data)
 
-      response = @http_client.exec(@http_request)
+      http_response = @http_client.exec(@http_request)
 
-      process_result(response)
+      process_result(http_response)
+    end
+
+    # Execute streaming HTTP request
+    def execute(&block : Crest::Response ->) : Nil
+      @http_client.set_proxy(@proxy)
+      @logger.request(self) if @logging
+
+      @http_request = new_http_request(@method, @url, @headers, @form_data)
+
+      @http_client.exec(@http_request) do |http_response|
+        response = process_result(http_response, &block)
+
+        if response
+          yield response
+        end
+      end
+    end
+
+    private def process_result(http_client_res) : Crest::Response
+      response = Response.new(http_client_res, self)
+      logger.response(response) if logging
+      response.return!
+    end
+
+    private def process_result(http_client_res, &block : Crest::Response ->)
+      response = Response.new(http_client_res, self)
+      logger.response(response) if logging
+      response.return!(&block)
     end
 
     # Convert `Request` object to cURL command
@@ -197,12 +231,6 @@ module Crest
       end
     end
 
-    private def process_result(http_client_res)
-      response = Response.new(http_client_res, self)
-      logger.response(response) if logging
-      response.return!
-    end
-
     private def parse_verb(method : String | Symbol) : String
       method.to_s.upcase
     end

src/crest/response.cr

@@ -5,7 +5,8 @@ require "../crest/redirector"
 module Crest
   # Response objects have several useful methods:
   #
-  # * `body`: The response body as a string
+  # * `body`: The response body as a String
+  # * `body_io`: The response body as a IO
   # * `status_code`: The HTTP response code
   # * `headers`: A hash of HTTP response headers
   # * `cookies`: A hash of HTTP cookies set by the server
@@ -16,10 +17,7 @@ module Crest
     getter http_client_res, request
 
     delegate body, to: http_client_res
-
-    def self.new(http_client_res : HTTP::Client::Response, request : Crest::Request)
-      self.new(http_client_res, request)
-    end
+    delegate body_io, to: http_client_res
 
     def initialize(@http_client_res : HTTP::Client::Response, @request : Crest::Request)
     end
@@ -29,6 +27,11 @@ module Crest
       redirector.follow
     end
 
+    def return!(&block : Crest::Response ->)
+      redirector = Redirector.new(self, request)
+      redirector.follow(&block)
+    end
+
     def url : String
       @request.url
     end