Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
A C based TokyoTyrant Ruby binding
branch: 59b5c253417eb9…

This branch is 1 commit ahead, 32 commits behind master

Fetching latest commit…

Cannot retrieve the latest commit at this time

Failed to load latest commit information.
benchmarks
ext
spec
.gitignore
COPYING
README.rdoc
Rakefile
ruby-tokyotyrant.gemspec

README.rdoc

TokyoTyrant Ruby Client

This is a c extension for Ruby to access TokyoTyrant databases. It currently supports key/value, table databases and table queries.

Install

# install tokyocabinet (1.4.34) and tokyotyrant (requires 1.1.35)
# after installing tc and tt on linux I had to /sbin/ldconfig (as root)

sudo gem install gemcutter
sudo gem tumble
sudo gem install ruby-tokyotyrant

# With Snow Leopard
sudo ARCHFLAGS="-arch x86_64" gem install ruby-tokyotyrant

Performance

This is not in production but the initial benchmarks are very interesting. Results look closer to the memcached gem than any other tyrant client I've seen for Ruby.

Example

Hash DB

# start tyrant like so:
# ttserver example.tch

require 'tokyo_tyrant'
db = TokyoTyrant::DB.new('127.0.0.1', 1978)

db['foo'] = 'Bar' # => "Bar"
db['foo']         # => "Bar"

db.each{ |k,v| puts [k, v].inspect }
# ["foo", "Bar"]
# => nil

db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}

Table DB

# start tyrant like so:
# ttserver example.tct

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t['bar'] = { :baz => 'box' } # => { :baz => 'box' }
t['bar']                     # => { :baz => 'box' }

t.each{ |k,v| puts [k, v].inspect }
# ["bar", {:baz=>"box"}]
# => nil

# bulk operations
h = {}
100.times do |i|
  h[i] = { :name => 'Pat', :sex => i % 2 > 0 ? 'male' : 'female' }
end
t.mput(h)
t.mget(0..3)
# => {"0"=>{:name=>"Pat", :sex=>"female"}, "1"=>{:name=>"Pat", :sex=>"male"}, "2"=>{:name=>"Pat", :sex=>"female"}, "3"=>{:name=>"Pat", :sex=>"male"}}

Table Query

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

100.times do |i|
  t[i] = { 'name' => "Pat #{i}", 'sex' => i % 2 > 0 ? 'male' : 'female' }
end

q = t.query
q.condition('sex', :streq, 'male')
q.limit(5)
# Get a list of IDs
ids = q.search
# => ["1", "3", "5", "7", "9"]
q.order_by(:name, :strdesc)
ids = q.search
# => ["99", "97", "95", "93", "91"]
# Get the actual records
q.get
# => [{:__id=>"99", :sex=>"male", :name=>"Pat 99"}, {:__id=>"97", :sex=>"male", :name=>"Pat 97"}, {:__id=>"95", :sex=>"male", :name=>"Pat 95"}, {:__id=>"93", :sex=>"male", :name=>"Pat 93"}, {:__id=>"91", :sex=>"male", :name=>"Pat 91"}]

# Alternative Syntax (better)

# Query using a block
t.query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => ["1", "3", "5", "7", "9"]

# Get records for a query
t.find{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

# Prepare but don't run a search, return a prepared query object
q = t.prepare_query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => #<TokyoTyrant::Query:0x247c14 @rdb=#<Object:0x2800a0>, @rdbquery=#<Object:0x247c00>>
q.search
# => ["1", "3", "5", "7", "9"]
q.get
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

Full Text Search

require 'tokyo_tyrant'
require 'nokogiri'
require 'open-uri'

t = TokyoTyrant::Table.new('127.0.0.1', 1978)

(1..13).each do |n|
  doc = Nokogiri::HTML(open("http://www.sacred-texts.com/chr/herm/hermes#{n}.htm"))
  chapter = doc.css('h2').last.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
  doc.css('p').each_with_index do |paragraph, i|
    paragraph = paragraph.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
    key = "chapter:#{n}:paragraph:#{i+1}"
    t[key] = { :chapter => chapter, :paragraph => paragraph }
  end
end

# full-text search with the phrase of
t.query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "chapter:13:paragraph:69", "chapter:13:paragraph:125"]

# full-text search with all tokens in
t.query{ |q| q.condition(:paragraph, :ftsand, 'logos word') }
# => ["chapter:1:paragraph:12", "chapter:1:paragraph:14", "chapter:1:paragraph:17", "chapter:1:paragraph:19", "chapter:1:paragraph:24", "chapter:1:paragraph:27", "chapter:1:paragraph:43", "chapter:1:paragraph:53", "... lots more ..."]

# full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, :ftsor, 'sermon key') }
# => ["chapter:5:paragraph:1", "chapter:9:paragraph:3", "chapter:10:paragraph:1", "chapter:10:paragraph:4", "chapter:10:paragraph:28", "chapter:11:paragraph:3", "chapter:11:paragraph:66", "chapter:11:paragraph:69", "... lots more ..."]

# negated full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, '!ftsor', 'the god he and I that said') }
# => ["chapter:1:paragraph:95", "chapter:1:paragraph:96", "chapter:1:paragraph:97", "chapter:1:paragraph:98", "chapter:1:paragraph:99", "chapter:2:paragraph:3", "chapter:2:paragraph:5", "chapter:2:paragraph:6", "... lots more ..."]

Meta Search (Multi Query)

query1 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
query2 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'logos') }

# Get the union of two query sets (OR)
t.search(:union, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "... lots more ..."]

# Get the intersection of two query sets (AND)
t.search(:intersection, query1, query2)
# => ["chapter:13:paragraph:5", "chapter:13:paragraph:44", "chapter:13:paragraph:69"]

# Get the difference of two query sets (ANDNOT)
t.search(:diff, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:57", "chapter:13:paragraph:125"]

Parallel Querying (Take that scalability!)

require 'tokyo_tyrant'
require 'faker'
tyrants = { 1 => TokyoTyrant::Table.new('127.0.0.1', 1978),
            2 => TokyoTyrant::Table.new('127.0.0.1', 1979) }

# dummy data
tyrants.each{ |account_id, table|
  table.clear
  20.times do |i|
    table["#{account_id}/#{i}"] = { # consistent hashing would be good here
      :name => Faker::Company.name,
      :plan => rand(3),
    }
  end
}

queries = tyrants.collect{ |account_id, table|
  table.prepare_query{ |q| q.condition 'plan', :numlt, '1' }
}

TokyoTyrant::Query.parallel_search(*queries).collect{ |r| {r[""] => r["name"]} }
# => [{"1/3"=>"Zemlak-Jerde"}, {"1/6"=>"O'Conner-Batz"}, {"1/9"=>"Kutch, Erdman and Aufderhar"}, {"1/11"=>"Bartoletti, Armstrong and Barrows"}, {"1/12"=>"Ferry-Dicki"}, {"2/0"=>"Schultz-O'Hara"}, {"2/1"=>"Emmerich, Feest and Huels"}, {"2/2"=>"Borer and Sons"}, {"2/3"=>"D'Amore Inc"}, {"2/5"=>"Koch and Sons"}, {"2/8"=>"Schaefer Group"}, {"2/11"=>"Stroman, Toy and Abernathy"}, {"2/19"=>"Gaylord, Reinger and White"}]

Lua Extension

# ttserver -ext spec/ext.lua 
require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t.run(:echo, 'hello', 'world')
# => "hello\tworld"

Contributors

  • Flinn Mueller (actsasflinn) author/maintainer

  • Justin Reagor (cheapRoc) specs

  • Seth Yates (sethyates) run method (lua ext)

  • John Mettraux (jmettraux) inspiration (rufus-tokyo)

Something went wrong with that request. Please try again.