Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Replace buftok due to incorrect delimiter handling

  • Loading branch information...
commit 88940b8a0cbc75483e0bee0a878c6960a6dd62fd 1 parent ae11102
@eagletmt authored
View
22 lib/twitter/streaming/buffered_tokenizer.rb
@@ -0,0 +1,22 @@
+module Twitter
+ module Streaming
+ class BufferedTokenizer
+ def initialize(delimiter)
+ @delimiter = delimiter
+ @buffer = ""
+ end
+
+ def extract(data)
+ @buffer << data
+ items = @buffer.split(@delimiter)
+ if @buffer.end_with?(@delimiter)
+ @buffer.clear
+ items
+ else
+ @buffer = items.pop
+ items
+ end
+ end
+ end
+ end
+end
View
2  lib/twitter/streaming/response.rb
@@ -1,4 +1,4 @@
-require 'buftok'
+require 'twitter/streaming/buffered_tokenizer'
module Twitter
module Streaming
View
25 spec/twitter/streaming/buffered_tokenizer_spec.rb
@@ -0,0 +1,25 @@
+require 'helper'
+
+describe Twitter::Streaming::BufferedTokenizer do
+ let(:tokenizer) { described_class.new("\r\n") }
+
+ describe '#extract' do
+ it 'returns an empty array when no delimiter is given' do
+ expect(tokenizer.extract("foo bar")).to be_empty
+ end
+
+ it 'returns a token' do
+ expect(tokenizer.extract("foo\r\n")).to eq ["foo"]
+ end
+
+ it 'returns multiple tokens' do
+ expect(tokenizer.extract("foo")).to be_empty
+ expect(tokenizer.extract("bar\r\nbaz\r\n")).to eq ["foobar", "baz"]
+ end
+
+ it 'handles splitted delimiter' do
+ expect(tokenizer.extract("foo\r")).to be_empty
+ expect(tokenizer.extract("\n")).to eq ["foo"]
+ end
+ end
+end
View
1  twitter.gemspec
@@ -3,7 +3,6 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
require 'twitter/version'
Gem::Specification.new do |spec|
- spec.add_dependency 'buftok', '~> 0.1.0'
spec.add_dependency 'descendants_tracker', '~> 0.0.1'
spec.add_dependency 'equalizer', '~> 0.0.7'
spec.add_dependency 'faraday', ['>= 0.8', '< 0.10']
Please sign in to comment.
Something went wrong with that request. Please try again.