Permalink
Browse files

Replace buftok due to incorrect delimiter handling

  • Loading branch information...
1 parent ae11102 commit 88940b8a0cbc75483e0bee0a878c6960a6dd62fd @eagletmt eagletmt committed Nov 20, 2013
@@ -0,0 +1,22 @@
+module Twitter
+ module Streaming
+ class BufferedTokenizer
+ def initialize(delimiter)
+ @delimiter = delimiter
+ @buffer = ""
+ end
+
+ def extract(data)
+ @buffer << data
+ items = @buffer.split(@delimiter)
+ if @buffer.end_with?(@delimiter)
+ @buffer.clear
+ items
+ else
+ @buffer = items.pop
+ items
+ end
+ end
+ end
+ end
+end
@@ -1,4 +1,4 @@
-require 'buftok'
+require 'twitter/streaming/buffered_tokenizer'
module Twitter
module Streaming
@@ -0,0 +1,25 @@
+require 'helper'
+
+describe Twitter::Streaming::BufferedTokenizer do
+ let(:tokenizer) { described_class.new("\r\n") }
+
+ describe '#extract' do
+ it 'returns an empty array when no delimiter is given' do
+ expect(tokenizer.extract("foo bar")).to be_empty
+ end
+
+ it 'returns a token' do
+ expect(tokenizer.extract("foo\r\n")).to eq ["foo"]
+ end
+
+ it 'returns multiple tokens' do
+ expect(tokenizer.extract("foo")).to be_empty
+ expect(tokenizer.extract("bar\r\nbaz\r\n")).to eq ["foobar", "baz"]
+ end
+
+ it 'handles splitted delimiter' do
+ expect(tokenizer.extract("foo\r")).to be_empty
+ expect(tokenizer.extract("\n")).to eq ["foo"]
+ end
+ end
+end
View
@@ -3,7 +3,6 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
require 'twitter/version'
Gem::Specification.new do |spec|
- spec.add_dependency 'buftok', '~> 0.1.0'
spec.add_dependency 'descendants_tracker', '~> 0.0.1'
spec.add_dependency 'equalizer', '~> 0.0.7'
spec.add_dependency 'faraday', ['>= 0.8', '< 0.10']

0 comments on commit 88940b8

Please sign in to comment.