Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Initial commit

  • Loading branch information...
commit a8cb808dd3583f09f40c94f4fb44d0de071f40e8 0 parents
@rkh authored
3  .gitmodules
@@ -0,0 +1,3 @@
+[submodule "vendor/tnetstring-rb"]
+ path = vendor/tnetstring-rb
+ url = https://github.com/mattyoho/tnetstring-rb.git
19 LICENSE
@@ -0,0 +1,19 @@
+Copyright (c) 2011 Konstantin Haase
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
54 README.md
@@ -0,0 +1,54 @@
+# OTNetstring
+
+Similar to [Tagged Netstrings](http://tnetstrings.org/), but optimized for streaming.
+
+What changed: The type info is not at the end of the encoded data, but at the beginning, right after
+the size info. That way nested objects can be created while reading from the stream. That way it is
+rather similar to [Bencode](http://en.wikipedia.org/wiki/Bencode), plus the Nestring advantage of
+always knowing how many bytes to read.
+
+Objects look like this:
+
+ Ruby TNetstring OTNetstring
+ 42 2:42# 2#42
+ "hi" 2:hi, 2,hi
+ true 4:true! 4!true
+ [1] 4:1:1#} 3:{1#1
+ {"a" => "b"} 8:1:a,=1:b,} 6{1,a1,b
+
+Similar implementations (both pure ruby, using recursion for nested objects) show the performance
+difference, esp. when simulating a network IO:
+
+ user system total real
+ TNetstring: simple objects 0.000000 0.000000 0.000000 ( 0.000160)
+ OTNetstring: simple objects 0.000000 0.000000 0.000000 ( 0.000166)*
+ TNetstring: long string 0.130000 0.050000 0.180000 ( 0.186946)
+ OTNetstring: long string 0.050000 0.000000 0.050000 ( 0.047778)
+ TNetstring: flat arrays 2.280000 2.940000 5.220000 ( 5.432791)
+ OTNetstring: flat arrays 0.470000 0.010000 0.480000 ( 0.484575)
+ TNetstring: complex nesting 4.310000 3.100000 7.410000 ( 7.562399)
+ OTNetstring: complex nesting 1.940000 0.010000 1.950000 ( 1.943385)
+ TNetstring: with remainder 0.300000 0.250000 0.550000 ( 0.574956)
+ OTNetstring: with remainder 0.080000 0.080000 0.160000 ( 0.152838)
+ TNetstring: streaming (3 GBit/sec) 2.160000 3.010000 5.170000 ( 5.180267)
+ OTNetstring: streaming (3 GBit/sec) 0.860000 0.050000 0.910000 ( 0.917463)
+ TNetstring: streaming (3 GBit/sec), with remainder 0.490000 0.360000 0.850000 ( 4.836673)
+ OTNetstring: streaming (3 GBit/sec), with remainder 0.080000 0.070000 0.150000 ( 0.148301)
+
+ * this is (insignificantly) slower, since OTNetstring wraps each String in a StringIO
+
+API is identical to [tnetstring-rb](https://github.com/mattyoho/tnetstring-rb), except that you use
+`OTNetstring` instead of `TNetstring` and that `parse` also takes `IO` or `IO`-like objects as
+argument.
+
+## Running benchmarks
+
+ git submodule init
+ git submodule update
+ ./bench.rb
+
+## Stuff to think about
+
+* If `:` would identify strings rather than `,` and each object would end with a `,`, then Netstrings
+ and TNetstring-Strings would be valid OTNetstring objects.
+* Representations of `true` and `false` could be shortened.
66 bench.rb
@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby -I lib -I vendor/tnetstring-rb/lib
+require 'otnetstring'
+require 'tnetstring'
+require 'benchmark'
+require 'stringio'
+
+class SlowStream
+ def initialize(str)
+ @length = str.length
+ @io = StringIO.new(str)
+ end
+
+ def readchar
+ slow_down
+ @io.readchar
+ end
+
+ def read(n)
+ slow_down(n)
+ @io.read(n)
+ end
+
+ def to_s
+ read(@length)
+ end
+
+ def split(*args)
+ to_s.split(*args)
+ end
+
+ def pos
+ @io.pos
+ end
+
+ def slow_down(times = 1)
+ # simulating network with 3 GBit/sec!
+ sleep(2.0e-08 * times)
+ end
+end
+
+def report(x, desc, *objects)
+ [TNetstring, OTNetstring].each do |type|
+ input = objects.map { |e| type.encode(e) }
+ x.report("#{type}: #{desc}") do
+ input.each { |data| type.parse(block_given? ? yield(data) : data) }
+ end
+ end
+end
+
+Benchmark.bmbm do |x|
+ simple = [0, nil, true, false, 42, {}, [], "hi"]
+ nested = simple
+ large = "x"*99999999
+ 3.times do
+ nested = {'a' => nested }
+ 1.upto(5000).each { |i| nested[i.to_s] = simple }
+ end
+
+ report(x, "simple objects", *simple)
+ report(x, "long string", large)
+ report(x, "flat arrays", simple*5000)
+ report(x, "complex nesting", nested)
+ report(x, "with remainder", simple) { |e| e << large }
+ report(x, "streaming (3 GBit/sec)", simple*5000) { |e| SlowStream.new(e) }
+ report(x, "streaming (3 GBit/sec), with remainder", simple) { |e| SlowStream.new(e << large) }
+end
44 lib/otnetstring.rb
@@ -0,0 +1,44 @@
+require 'stringio'
+
+module OTNetstring
+ def self.parse(io)
+ io = StringIO.new(io) if io.respond_to? :to_str
+ length, byte = "", "0"
+ while byte =~ /\d/
+ length << byte
+ byte = io.readchar
+ end
+ length = length.to_i
+ case byte
+ when '#' then Integer io.read(length)
+ when ',' then io.read(length)
+ when '~' then nil
+ when '!' then io.read(length) == 'true'
+ when '[', '{'
+ hash = byte == "{"
+ object = hash ? {} : []
+ start = io.pos
+ while io.pos - start < length
+ value = parse(io)
+ if hash
+ object[value] = parse(io)
+ else
+ object << value
+ end
+ end
+ object
+ end
+ end
+
+ def self.encode(obj, string_sep = ',')
+ case obj
+ when String then "#{obj.length}#{string_sep}#{obj}"
+ when Integer then encode(obj.inspect, '#')
+ when NilClass then "0~"
+ when Array then encode(obj.map { |e| encode(e) }.join, '[')
+ when Hash then encode(obj.map { |a,b| encode(a)+encode(b) }.join, '{')
+ when FalseClass, TrueClass then encode(obj.inspect, '!')
+ else fail 'cannot encode %p' % obj
+ end
+ end
+end
119 spec/otnetstring_spec.rb
@@ -0,0 +1,119 @@
+# Based on tnetstring-rb's spec/tnetstring_spec.rb
+#
+# Copyright (c) 2011 Matt Yoho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'otnetstring'
+
+describe OTNetstring do
+ context "parsing" do
+ it "parses an integer" do
+ OTNetstring.parse('5#12345').should == 12345
+ end
+
+ it "parses an empty string" do
+ OTNetstring.parse('0,').should == ""
+ end
+
+ it "parses a string" do
+ OTNetstring.parse('12,this is cool').should == "this is cool"
+ end
+
+ it "parses to an empty array" do
+ OTNetstring.parse('0[').should == []
+ end
+
+ it "parses an arbitrary array of ints and strings" do
+ OTNetstring.parse('21[5#123455#678905,xxxxx').should == [12345, 67890, 'xxxxx']
+ end
+
+ it "parses to an empty hash" do
+ OTNetstring.parse('0{').should == {}
+ end
+
+ it "parses an arbitrary hash of ints, strings, and arrays" do
+ OTNetstring.parse('30{5,hello20[11#123456789014,this').should == {"hello" => [12345678901, 'this']}
+ end
+
+ it "parses a null" do
+ OTNetstring.parse('0~').should == nil
+ end
+
+ it "parses a boolean" do
+ OTNetstring.parse('4!true!').should == true
+ end
+ end
+
+ context "encoding" do
+ it "encodes an integer" do
+ OTNetstring.encode(42).should == "2#42"
+ end
+
+ it "encodes a string" do
+ OTNetstring.encode("hello world").should == "11,hello world"
+ end
+
+ context "boolean" do
+ it "encodes true as 'true'" do
+ OTNetstring.encode(true).should == "4!true"
+ end
+
+ it "encodes false as 'false'" do
+ OTNetstring.encode(false).should == "5!false"
+ end
+ end
+
+ it "encodes nil" do
+ OTNetstring.encode(nil).should == "0~"
+ end
+
+ context "arrays" do
+ it "encodes an empty array" do
+ OTNetstring.encode([]).should == "0["
+ end
+
+ it "encodes an array of arbitrary elements" do
+ OTNetstring.encode(["cat", false, 123]).should == "17[3,cat5!false3#123"
+ end
+
+ it "encodes nested arrays" do
+ OTNetstring.encode(["cat", [false, 123]]).should == "20[3,cat12[5!false3#123"
+ end
+ end
+
+ context "hashes" do
+ it "encodes an empty hash" do
+ OTNetstring.encode({}).should == "0{"
+ end
+
+ it "encodes an arbitrary hash of primitives and arrays" do
+ OTNetstring.encode({"hello" => [12345678901, 'this']}).should == '30{5,hello20[11#123456789014,this'
+ end
+
+ it "encodes nested hashes" do
+ OTNetstring.encode({"hello" => {"world" => 42}}).should == '21{5,hello11{5,world2#42'
+ end
+ end
+
+ it "rejects non-primitives" do
+ expect { TNetstring.encode(Object.new) }.to raise_error
+ end
+ end
+end
1  vendor/tnetstring-rb
@@ -0,0 +1 @@
+Subproject commit 9cda07aba8e290fc09c5c4cc39fe66c985f3ded8
Please sign in to comment.
Something went wrong with that request. Please try again.