TokyoTyrant Ruby Client

This is a c extension for Ruby to access TokyoTyrant databases. It currently supports key/value, table databases and table queries.

Install

  # install tokyocabinet (1.4.34) and tokyotyrant (requires 1.1.35)
  # after installing tc and tt on linux I had to /sbin/ldconfig (as root)
  sudo gem install gemcutter
  sudo gem tumble
  sudo gem install ruby-tokyotyrant

Performance

This is not in production but the initial benchmarks are very interesting. Results look closer to the memcached gem than any other tyrant client I’ve seen for Ruby.

• Key/Value Store: gist.github.com/75212
• Table Store: gist.github.com/74116
• Bulk Operations: gist.github.com/83194
• Bulk Table Operations: gist.github.com/87215

Example

Hash DB

  # start tyrant like so:
  # ttserver example.tch

  require 'tokyo_tyrant'
  db = TokyoTyrant::DB.new('127.0.0.1', 1978)

  db['foo'] = 'Bar' # => "Bar"
  db['foo']         # => "Bar"

  db.each{ |k,v| puts [k, v].inspect }
  # ["foo", "Bar"]
  # => nil

  db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
  db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}

Table DB

  # start tyrant like so:
  # ttserver example.tct

  require 'tokyo_tyrant'
  t = TokyoTyrant::Table.new('127.0.0.1', 1978)

  t['bar'] = { :baz => 'box' } # => { :baz => 'box' }
  t['bar']                     # => { :baz => 'box' }

  t.each{ |k,v| puts [k, v].inspect }
  # ["bar", {:baz=>"box"}]
  # => nil

  # bulk operations
  h = {}
  100.times do |i|
    h[i] = { :name => 'Pat', :sex => i % 2 > 0 ? 'male' : 'female' }
  end
  t.mput(h)
  t.mget(0..3)
  # => {"0"=>{:name=>"Pat", :sex=>"female"}, "1"=>{:name=>"Pat", :sex=>"male"}, "2"=>{:name=>"Pat", :sex=>"female"}, "3"=>{:name=>"Pat", :sex=>"male"}}

Table Query

  require 'tokyo_tyrant'
  t = TokyoTyrant::Table.new('127.0.0.1', 1978)

  100.times do |i|
    t[i] = { 'name' => "Pat #{i}", 'sex' => i % 2 > 0 ? 'male' : 'female' }
  end

  q = t.query
  q.condition('sex', :streq, 'male')
  q.limit(5)
  # Get a list of IDs
  ids = q.search
  # => ["1", "3", "5", "7", "9"]
  q.order_by(:name, :strdesc)
  ids = q.search
  # => ["99", "97", "95", "93", "91"]
  # Get the actual records
  q.get
  # => [{:__id=>"99", :sex=>"male", :name=>"Pat 99"}, {:__id=>"97", :sex=>"male", :name=>"Pat 97"}, {:__id=>"95", :sex=>"male", :name=>"Pat 95"}, {:__id=>"93", :sex=>"male", :name=>"Pat 93"}, {:__id=>"91", :sex=>"male", :name=>"Pat 91"}]

  # Alternative Syntax (better)

  # Query using a block
  t.query{ |q|
    q.condition('sex', :streq, 'male')
    q.limit(5)
  }
  # => ["1", "3", "5", "7", "9"]

  # Get records for a query
  t.find{ |q|
    q.condition('sex', :streq, 'male')
    q.limit(5)
  }
  # => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

  # Prepare but don't run a search, return a prepared query object
  q = t.prepare_query{ |q|
    q.condition('sex', :streq, 'male')
    q.limit(5)
  }
  # => #<TokyoTyrant::Query:0x247c14 @rdb=#<Object:0x2800a0>, @rdbquery=#<Object:0x247c00>>
  q.search
  # => ["1", "3", "5", "7", "9"]
  q.get
  # => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

Full Text Search

  require 'tokyo_tyrant'
  require 'nokogiri'
  require 'open-uri'

  t = TokyoTyrant::Table.new('127.0.0.1', 1978)

  (1..13).each do |n|
    doc = Nokogiri::HTML(open("http://www.sacred-texts.com/chr/herm/hermes#{n}.htm"))
    chapter = doc.css('h2').last.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
    doc.css('p').each_with_index do |paragraph, i|
      paragraph = paragraph.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
      key = "chapter:#{n}:paragraph:#{i+1}"
      t[key] = { :chapter => chapter, :paragraph => paragraph }
    end
  end

  # full-text search with the phrase of
  t.query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
  # => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "chapter:13:paragraph:69", "chapter:13:paragraph:125"]

  # full-text search with all tokens in
  t.query{ |q| q.condition(:paragraph, :ftsand, 'logos word') }
  # => ["chapter:1:paragraph:12", "chapter:1:paragraph:14", "chapter:1:paragraph:17", "chapter:1:paragraph:19", "chapter:1:paragraph:24", "chapter:1:paragraph:27", "chapter:1:paragraph:43", "chapter:1:paragraph:53", "... lots more ..."]

  # full-text search with at least one token in
  t.query{ |q| q.condition(:paragraph, :ftsor, 'sermon key') }
  # => ["chapter:5:paragraph:1", "chapter:9:paragraph:3", "chapter:10:paragraph:1", "chapter:10:paragraph:4", "chapter:10:paragraph:28", "chapter:11:paragraph:3", "chapter:11:paragraph:66", "chapter:11:paragraph:69", "... lots more ..."]

  # negated full-text search with at least one token in
  t.query{ |q| q.condition(:paragraph, '!ftsor', 'the god he and I that said') }
  # => ["chapter:1:paragraph:95", "chapter:1:paragraph:96", "chapter:1:paragraph:97", "chapter:1:paragraph:98", "chapter:1:paragraph:99", "chapter:2:paragraph:3", "chapter:2:paragraph:5", "chapter:2:paragraph:6", "... lots more ..."]

Meta Search (Multi Query)

  query1 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
  query2 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'logos') }

  # Get the union of two query sets (OR)
  t.search(:union, query1, query2)
  # => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "... lots more ..."]

  # Get the intersection of two query sets (AND)
  t.search(:intersection, query1, query2)
  # => ["chapter:13:paragraph:5", "chapter:13:paragraph:44", "chapter:13:paragraph:69"]

  # Get the difference of two query sets (ANDNOT)
  t.search(:diff, query1, query2)
  # => ["chapter:13:paragraph:4", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:57", "chapter:13:paragraph:125"]

Parallel Querying (Take that scalability!)

  require 'tokyo_tyrant'
  require 'faker'
  tyrants = { 1 => TokyoTyrant::Table.new('127.0.0.1', 1978),
              2 => TokyoTyrant::Table.new('127.0.0.1', 1979) }

  # dummy data
  tyrants.each{ |account_id, table|
    table.clear
    20.times do |i|
      table["#{account_id}/#{i}"] = { # consistent hashing would be good here
        :name => Faker::Company.name,
        :plan => rand(3),
      }
    end
  }

  queries = tyrants.collect{ |account_id, table|
    table.prepare_query{ |q| q.condition 'plan', :numlt, '1' }
  }

  TokyoTyrant::Query.parallel_search(*queries).collect{ |r| {r[""] => r["name"]} }
  # => [{"1/3"=>"Zemlak-Jerde"}, {"1/6"=>"O'Conner-Batz"}, {"1/9"=>"Kutch, Erdman and Aufderhar"}, {"1/11"=>"Bartoletti, Armstrong and Barrows"}, {"1/12"=>"Ferry-Dicki"}, {"2/0"=>"Schultz-O'Hara"}, {"2/1"=>"Emmerich, Feest and Huels"}, {"2/2"=>"Borer and Sons"}, {"2/3"=>"D'Amore Inc"}, {"2/5"=>"Koch and Sons"}, {"2/8"=>"Schaefer Group"}, {"2/11"=>"Stroman, Toy and Abernathy"}, {"2/19"=>"Gaylord, Reinger and White"}]

Lua Extension

  # ttserver -ext spec/ext.lua
  require 'tokyo_tyrant'
  t = TokyoTyrant::Table.new('127.0.0.1', 1978)

  t.run(:echo, 'hello', 'world')
  # => "hello\tworld"

Contributors

• Flinn Mueller (actsasflinn) author/maintainer
• Justin Reagor (cheapRoc) specs
• Seth Yates (sethyates) run method (lua ext)
• John Mettraux (jmettraux) inspiration (rufus-tokyo)