Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Refactor reading and chunked reading into separate objects and test them

  • Loading branch information...
commit b17e25f9ea8abf61f52bd6537aca6b47fcc11f4e 1 parent 5103d5f
@wycats wycats authored
Showing with 291 additions and 10 deletions.
  1. +115 −0 lib/net2/http/readers.rb
  2. +27 −10 lib/net2/http/response.rb
  3. +149 −0 test/test_reader.rb
View
115 lib/net2/http/readers.rb
@@ -0,0 +1,115 @@
+module Net2
+ class HTTP
+ class BodyReader
+ def initialize(socket, endpoint, content_length)
+ @socket = socket
+ @endpoint = endpoint
+ @content_length = content_length
+
+ @read = 0
+ end
+
+ def read_to_endpoint(len=@content_length)
+ remain = @content_length - @read
+
+ raise EOFError if remain.zero?
+ len = remain if len > remain
+
+ begin
+ output = @socket.read_nonblock(len)
+ @endpoint << output
+ @read += output.size
+ rescue Errno::EWOULDBLOCK, Errno::EAGAIN, OpenSSL::SSL::SSLError
+ end
+ end
+
+ def wait(timeout=nil)
+ if @io.is_a?(OpenSSL::SSL::SSLSocket)
+ return if IO.select nil, [@io], nil, timeout
+ else
+ return if IO.select [@io], nil, nil, timeout
+ end
+
+ raise Timeout::Error
+ end
+ end
+
+ class ChunkedBodyReader
+ BUFSIZE = 1024
+
+ def initialize(socket, endpoint)
+ @socket = socket
+ @endpoint = endpoint
+ @raw_buffer = ""
+ @out_buffer = ""
+
+ @chunk_size = nil
+ @state = :process_size
+
+ @handled_trailer = false
+ end
+
+ def read_to_endpoint(len=nil)
+ fill_buffer
+
+ send @state
+
+ if !len
+ @endpoint << @out_buffer
+ @out_buffer = ""
+ elsif @out_buffer.size > len
+ @endpoint << @out_buffer.slice!(0, len)
+ else
+ @endpoint << @out_buffer
+ @out_buffer = ""
+ end
+ end
+
+ def process_size
+ idx = @raw_buffer.index("\r\n")
+ return unless idx
+
+ size_str = @raw_buffer.slice!(0, idx)
+ @raw_buffer.slice!(0, 2)
+
+ if size_str == "0"
+ @state = :process_trailer
+ process_trailer
+ return
+ end
+
+ @size = size_str.to_i(16)
+
+ @state = :process_chunk
+ process_chunk
+ end
+
+ def process_chunk
+ if @raw_buffer.size > @size
+ @out_buffer << @raw_buffer.slice!(0, @size)
+ @state = :process_size
+ process_size
+ else
+ @size -= @raw_buffer.size
+ @out_buffer << @raw_buffer
+ @raw_buffer = ""
+ end
+ end
+
+ # TODO: Make this handle trailers
+ def process_trailer
+ raise EOFError if @eof && @out_buffer.empty?
+ @eof = true
+ end
+
+ private
+ def fill_buffer
+ @raw_buffer << @socket.read_nonblock(BUFSIZE)
+ return true
+ rescue Errno::EWOULDBLOCK, EOFError
+ return false
+ end
+ end
+ end
+end
+
View
37 lib/net2/http/response.rb
@@ -1,5 +1,6 @@
require "net2/http/header"
require "net2/http/gzip"
+require "stringio"
module Net2
class HTTP
@@ -165,7 +166,7 @@ def read_body(dest = nil, &block)
raise IOError, "#{self.class}\#read_body called twice" if dest or block
return @body
end
- to = procdest(dest, block)
+ to = build_pipeline(dest, block)
stream_check
if body_exist?
read_body_0 to
@@ -319,23 +320,39 @@ def inflater
end
end
- def procdest(dest, block)
- raise ArgumentError, 'both arg and block given for HTTP method' \
- if dest and block
+ class StringAdapter
+ def initialize(buffer)
+ @buffer = buffer
+ end
+
+ def <<(chunk)
+ @buffer << chunk
+ end
+
+ def string
+ @buffer
+ end
+ end
+
+ def build_pipeline(dest, block)
+ if dest and block
+ raise ArgumentError, 'both arg and block given for HTTP method' \
+ end
+
if block
- wrapped_dest = ReadAdapter.new(block)
+ endpoint = ReadAdapter.new(block)
else
- wrapped_dest = dest || ''
+ endpoint = StringAdapter.new(dest || '')
end
case self["Content-Encoding"]
when "gzip"
- GzipAdapter.new(wrapped_dest)
+ endpoint = GzipAdapter.new(endpoint)
when "deflate"
- InflateAdapter.new(wrapped_dest)
- else
- wrapped_dest
+ endpoint = InflateAdapter.new(endpoint)
end
+
+ endpoint
end
end
View
149 test/test_reader.rb
@@ -0,0 +1,149 @@
+require "test/unit"
+require "utils"
+require "net2/http/readers"
+
+module Net2
+ class TestBodyReader < Test::Unit::TestCase
+ def setup
+ @body = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
+
+ @read, @write = IO.pipe
+ @buf = ""
+ @reader = Net2::HTTP::BodyReader.new(@read, @buf, @body.bytesize)
+ end
+
+ def teardown
+ @read.close
+ @write.close
+ end
+
+ def test_simple_read
+ @write.write @body
+ @reader.read_to_endpoint
+ assert_equal @body, @buf
+ end
+
+ def test_read_chunks
+ @write.write @body
+ @reader.read_to_endpoint 50
+ assert_equal @body.slice(0,50), @buf
+ end
+
+ def test_read_over
+ @write.write @body
+ @reader.read_to_endpoint 50
+ assert_equal @body.slice(0,50), @buf
+
+ @reader.read_to_endpoint @body.size
+ assert_equal @body, @buf
+
+ assert_raises EOFError do
+ @reader.read_to_endpoint 10
+ end
+ end
+
+ def test_blocking
+ @write.write @body.slice(0,50)
+ @reader.read_to_endpoint 100
+ assert_equal @body.slice(0,50), @buf
+
+ @reader.read_to_endpoint 100
+ assert_equal @body.slice(0,50), @buf
+
+ @write.write @body.slice(50..-1)
+ @reader.read_to_endpoint
+ assert_equal @body, @buf
+
+ assert_raises EOFError do
+ @reader.read_to_endpoint 10
+ end
+ end
+ end
+
+ class TestChunkedBodyReader < Test::Unit::TestCase
+ def setup
+ @body = "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
+
+ @read, @write = IO.pipe
+ @buf = ""
+ @reader = Net2::HTTP::ChunkedBodyReader.new(@read, @buf)
+ end
+
+ def teardown
+ @read.close
+ @write.close
+ end
+
+ def test_simple_read
+ @write.write "#{@body.size.to_s(16)}\r\n#{@body}\r\n0\r\n"
+ @reader.read_to_endpoint
+ assert_equal @body, @buf
+ end
+
+ def test_read_chunks
+ @write.write "#{@body.size.to_s(16)}\r\n#{@body}\r\n0\r\n"
+ @reader.read_to_endpoint 50
+ assert_equal @body.slice(0,50), @buf
+ end
+
+ def test_read_over
+ @write.write "#{@body.size.to_s(16)}\r\n#{@body}\r\n0\r\n"
+ @reader.read_to_endpoint 50
+ assert_equal @body.slice(0,50), @buf
+
+ @reader.read_to_endpoint @body.size
+ assert_equal @body, @buf
+
+ assert_raises EOFError do
+ @reader.read_to_endpoint 10
+ end
+ end
+
+ def test_blocking
+ size = @body.size.to_s(16)
+ body = "#{size}\r\n#{@body}\r\n0\r\n"
+
+ @write.write body.slice(0,50 + size.size + 2)
+ @reader.read_to_endpoint 100
+ assert_equal @body.slice(0,50), @buf
+
+ @reader.read_to_endpoint 100
+ assert_equal @body.slice(0,50), @buf
+
+ @write.write body.slice((50 + size.size + 2)..-1)
+ @reader.read_to_endpoint
+ assert_equal @body, @buf
+
+ assert_raises EOFError do
+ @reader.read_to_endpoint 10
+ end
+ end
+
+ def test_multi_chunks
+ @write.write 50.to_s(16)
+ @write.write "\r\n"
+ @write.write @body.slice(0,50)
+
+ @reader.read_to_endpoint 100
+ assert_equal @body.slice(0,50), @buf
+
+ @write.write "\r\n"
+ rest = @body[50..-1]
+ @write.write rest.size.to_s(16)
+ @write.write "\r\n"
+ @write.write rest
+
+ @reader.read_to_endpoint
+ assert_equal @body, @buf
+
+ @write.write "\r\n0\r\n"
+ @reader.read_to_endpoint
+ assert_equal @body, @buf
+
+ assert_raises EOFError do
+ @reader.read_to_endpoint
+ end
+ end
+ end
+end
+
Please sign in to comment.
Something went wrong with that request. Please try again.