Permalink
Browse files

Fixes bug with #readlines

  • Loading branch information...
1 parent 964425b commit 5efa4896fa655b3fa692905ed88b0a00b86b4b82 @alindeman committed Oct 21, 2011
Showing with 89 additions and 10 deletions.
  1. +1 −0 ganapati.gemspec
  2. +10 −10 lib/ganapati/client.rb
  3. +71 −0 test/client_test.rb
  4. +7 −0 test/test_helper.rb
View
@@ -19,4 +19,5 @@ Gem::Specification.new do |s|
s.executables << 'hcp'
s.rubyforge_project = "ganapati"
s.add_dependency('thrift', '>= 0.5.0')
+ s.add_development_dependency('mocha', '>=0.10.0')
end
@@ -53,19 +53,19 @@ def readchunks(path, chunksize=1048576)
end
def readlines(path, sep="\n")
- lastbit = ""
+ buffer = ""
+
readchunks(path) { |chunk|
- parts = chunk.split(sep)
- if parts.length == 0
- yield lastbit if lastbit != ""
- elsif parts.length == 1
- yield lastbit + parts.first
- else
- yield lastbit + parts.first
- parts.slice(1, parts.length).each { |p| yield p }
+ buffer << chunk
+
+ # partitions[1] will be empty if sep does not exist in the string
+ while partitions = buffer.partition(sep) and !partitions[1].empty?
+ yield partitions.first
+ buffer = partitions.last
end
- lastbit = ""
}
+
+ yield buffer if buffer.size > 0
end
# for writing to a new file
View
@@ -0,0 +1,71 @@
+require "test_helper"
+
+class ClientTest < Test::Unit::TestCase
+ def stub_thrift!
+ @socket = stub_everything("socket")
+ Thrift::Socket.stubs(:new).returns(stub)
+
+ @transport = stub_everything("transport")
+ Thrift::BufferedTransport.stubs(:new).returns(@transport)
+
+ @protocol = stub_everything("protocol")
+ Thrift::BinaryProtocol.stubs(:new).returns(@protocol)
+
+ @client = stub_everything("client")
+ ThriftHadoopFileSystem::Client.stubs(:new).returns(@client)
+ end
+
+ def setup
+ stub_thrift!
+ end
+
+ def test_readlines_with_one_chunk
+ ganapati = Ganapati::Client.new("127.0.0.1", 1234)
+ ganapati.stubs(:readchunks).yields("abc123")
+
+ lines = []
+ ganapati.readlines("/foo") { |line| lines << line }
+
+ assert_equal ["abc123"], lines
+ end
+
+ def test_readlines_with_two_chunks
+ ganapati = Ganapati::Client.new("127.0.0.1", 1234)
+ ganapati.stubs(:readchunks).multiple_yields(["abc\n123"], ["456"])
+
+ lines = []
+ ganapati.readlines("/foo") { |line| lines << line }
+
+ assert_equal ["abc", "123456"], lines
+ end
+
+ def test_readlines_with_chunks_with_multiple_lines
+ ganapati = Ganapati::Client.new("127.0.0.1", 1234)
+ ganapati.stubs(:readchunks).multiple_yields(["abc\n123\n456\n"], ["789"])
+
+ lines = []
+ ganapati.readlines("/foo") { |line| lines << line }
+
+ assert_equal ["abc", "123", "456", "789"], lines
+ end
+
+ def test_readlines_with_middle_chunk
+ ganapati = Ganapati::Client.new("127.0.0.1", 1234)
+ ganapati.stubs(:readchunks).multiple_yields(["abc\n123"], ["456"], ["789"])
+
+ lines = []
+ ganapati.readlines("/foo") { |line| lines << line }
+
+ assert_equal ["abc", "123456789"], lines
+ end
+
+ def test_readlines_with_sep_character_at_the_end_of_chunks
+ ganapati = Ganapati::Client.new("127.0.0.1", 1234)
+ ganapati.stubs(:readchunks).multiple_yields(["abc\n"], ["123\n"], ["456\n"])
+
+ lines = []
+ ganapati.readlines("/foo") { |line| lines << line }
+
+ assert_equal ["abc", "123", "456"], lines
+ end
+end
View
@@ -0,0 +1,7 @@
+require "test/unit"
+
+require "rubygems"
+require "mocha"
+
+$LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
+require "ganapati"

0 comments on commit 5efa489

Please sign in to comment.