Permalink
Browse files

started work on secondary indexes

  • Loading branch information...
1 parent 780ea4c commit 5e65821bca237f49d8fcb61836631c5bae12288b @erwaller erwaller committed Jan 5, 2012
Showing with 27 additions and 9 deletions.
  1. +6 −2 lib/soulmate/base.rb
  2. +19 −5 lib/soulmate/loader.rb
  3. +2 −2 lib/soulmate/matcher.rb
@@ -14,12 +14,16 @@ def base
"soulmate-index:#{type}"
end
- def database
+ def data_base
"soulmate-data:#{type}"
end
- def cachebase
+ def cache_base
"soulmate-cache:#{type}"
end
+
+ def secondary_index_base
+ "soulmate-secondary-index:#{type}"
+ end
end
end
@@ -18,7 +18,7 @@ def load(items)
# everything will work itself out as soon as the cache expires again.
# delete the data stored for this type
- Soulmate.redis.del(database)
+ Soulmate.redis.del(data_base)
items.each_with_index do |item, i|
add(item, :skip_duplicate_check => true)
@@ -35,23 +35,37 @@ def add(item, opts = {})
Soulmate.redis.pipelined do
# store the raw data in a separate key to reduce memory usage
- Soulmate.redis.hset(database, item["id"], MultiJson.encode(item))
+ Soulmate.redis.hset(data_base, item["id"], MultiJson.encode(item))
+
phrase = ([item["term"]] + (item["aliases"] || [])).join(' ')
- prefixes_for_phrase(phrase).each do |p|
+ prefixes = prefixes_for_phrase(phrase)
+
+ # store the main index
+ prefixes.each do |p|
Soulmate.redis.sadd(base, p) # remember this prefix in a master set
Soulmate.redis.zadd("#{base}:#{p}", item["score"], item["id"]) # store the id of this term in the index
end
+
+ # store secondary indexes
+ indexes = item["indexes"] || []
+ indexes.each do |index|
+ Soulmate.redis.sadd(secondary_index_base, index) # remember this index
+ prefixes.each do |p|
+ Soulmate.redis.sadd("#{secondary_index_base}:#{index}", p) # remember this prefix in a master set for this secondary index
+ Soulmate.redis.zadd("#{secondary_index_base}:#{index}:#{p}", item["score"], item["id"]) # store the id of this term in the secondary index
+ end
+ end
end
end
# remove only cares about an item's id, but for consistency takes an object
def remove(item)
- prev_item = Soulmate.redis.hget(database, item["id"])
+ prev_item = Soulmate.redis.hget(data_base, item["id"])
if prev_item
prev_item = MultiJson.decode(prev_item)
# undo the operations done in add
Soulmate.redis.pipelined do
- Soulmate.redis.hdel(database, prev_item["id"])
+ Soulmate.redis.hdel(data_base, prev_item["id"])
phrase = ([prev_item["term"]] + (prev_item["aliases"] || [])).join(' ')
prefixes_for_phrase(phrase).each do |p|
Soulmate.redis.srem(base, p)
@@ -11,7 +11,7 @@ def matches_for_term(term, options = {})
return [] if words.empty?
- cachekey = "#{cachebase}:" + words.join('|')
+ cachekey = "#{cache_base}:" + words.join('|')
if !options[:cache] || !Soulmate.redis.exists(cachekey)
interkeys = words.map { |w| "#{base}:#{w}" }
@@ -21,7 +21,7 @@ def matches_for_term(term, options = {})
ids = Soulmate.redis.zrevrange(cachekey, 0, options[:limit] - 1)
if ids.size > 0
- results = Soulmate.redis.hmget(database, *ids)
+ results = Soulmate.redis.hmget(data_base, *ids)
results = results.reject{ |r| r.nil? } # handle cached results for ids which have since been deleted
results.map { |r| MultiJson.decode(r) }
else

0 comments on commit 5e65821

Please sign in to comment.