From 88940b8a0cbc75483e0bee0a878c6960a6dd62fd Mon Sep 17 00:00:00 2001
From: Kohei Suzuki <eagletmt@gmail.com>
Date: Wed, 20 Nov 2013 21:05:51 +0900
Subject: [PATCH] Replace buftok due to incorrect delimiter handling

---
 lib/twitter/streaming/buffered_tokenizer.rb   | 22 ++++++++++++++++
 lib/twitter/streaming/response.rb             |  2 +-
 .../streaming/buffered_tokenizer_spec.rb      | 25 +++++++++++++++++++
 twitter.gemspec                               |  1 -
 4 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 lib/twitter/streaming/buffered_tokenizer.rb
 create mode 100644 spec/twitter/streaming/buffered_tokenizer_spec.rb

diff --git a/lib/twitter/streaming/buffered_tokenizer.rb b/lib/twitter/streaming/buffered_tokenizer.rb
new file mode 100644
index 000000000..42681e19b
--- /dev/null
+++ b/lib/twitter/streaming/buffered_tokenizer.rb
@@ -0,0 +1,22 @@
+module Twitter
+  module Streaming
+    class BufferedTokenizer
+      def initialize(delimiter)
+        @delimiter = delimiter
+        @buffer = ""
+      end
+
+      def extract(data)
+        @buffer << data
+        items = @buffer.split(@delimiter)
+        if @buffer.end_with?(@delimiter)
+          @buffer.clear
+          items
+        else
+          @buffer = items.pop
+          items
+        end
+      end
+    end
+  end
+end
diff --git a/lib/twitter/streaming/response.rb b/lib/twitter/streaming/response.rb
index fd88ca6a8..6ef9cefb4 100644
--- a/lib/twitter/streaming/response.rb
+++ b/lib/twitter/streaming/response.rb
@@ -1,4 +1,4 @@
-require 'buftok'
+require 'twitter/streaming/buffered_tokenizer'
 
 module Twitter
   module Streaming
diff --git a/spec/twitter/streaming/buffered_tokenizer_spec.rb b/spec/twitter/streaming/buffered_tokenizer_spec.rb
new file mode 100644
index 000000000..8e6c1e646
--- /dev/null
+++ b/spec/twitter/streaming/buffered_tokenizer_spec.rb
@@ -0,0 +1,25 @@
+require 'helper'
+
+describe Twitter::Streaming::BufferedTokenizer do
+  let(:tokenizer) { described_class.new("\r\n") }
+
+  describe '#extract' do
+    it 'returns an empty array when no delimiter is given' do
+      expect(tokenizer.extract("foo bar")).to be_empty
+    end
+
+    it 'returns a token' do
+      expect(tokenizer.extract("foo\r\n")).to eq ["foo"]
+    end
+
+    it 'returns multiple tokens' do
+      expect(tokenizer.extract("foo")).to be_empty
+      expect(tokenizer.extract("bar\r\nbaz\r\n")).to eq ["foobar", "baz"]
+    end
+
+    it 'handles splitted delimiter' do
+      expect(tokenizer.extract("foo\r")).to be_empty
+      expect(tokenizer.extract("\n")).to eq ["foo"]
+    end
+  end
+end
diff --git a/twitter.gemspec b/twitter.gemspec
index da6b53fb8..72cd3ccec 100644
--- a/twitter.gemspec
+++ b/twitter.gemspec
@@ -3,7 +3,6 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'twitter/version'
 
 Gem::Specification.new do |spec|
-  spec.add_dependency 'buftok', '~> 0.1.0'
   spec.add_dependency 'descendants_tracker', '~> 0.0.1'
   spec.add_dependency 'equalizer', '~> 0.0.7'
   spec.add_dependency 'faraday', ['>= 0.8', '< 0.10']